You are viewing a plain text version of this content. The canonical link for it is here.
Posted to builds@solr.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2022/04/05 12:36:25 UTC

[JENKINS] Solr-main-MacOSX (64bit/jdk-16.0.2) - Build # 860 - Still Unstable!

Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/860/
Java: 64bit/jdk-16.0.2 -XX:+UseCompressedOops -XX:+UseSerialGC

2 tests failed.
FAILED:  org.apache.solr.search.facet.TestJsonFacetRefinement.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.search.facet.TestJsonFacetRefinement: 
   1) Thread[id=7925, name=Session-HouseKeeper-4d49867c-1, state=TIMED_WAITING, group=TGRP-TestJsonFacetRefinement]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.search.facet.TestJsonFacetRefinement: 
   1) Thread[id=7925, name=Session-HouseKeeper-4d49867c-1, state=TIMED_WAITING, group=TGRP-TestJsonFacetRefinement]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
	at __randomizedtesting.SeedInfo.seed([C3499354DD82270F]:0)


FAILED:  org.apache.solr.search.join.CrossCollectionJoinQueryTest.classMethod

Error Message:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:50692/solr: Underlying core creation failed while creating collection: parts

Stack Trace:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:50692/solr: Underlying core creation failed while creating collection: parts
	at __randomizedtesting.SeedInfo.seed([C3499354DD82270F]:0)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:715)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:405)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:371)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1248)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:969)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:896)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:223)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:238)
	at app//org.apache.solr.search.join.CrossCollectionJoinQueryTest.setupCluster(CrossCollectionJoinQueryTest.java:60)
	at java.base@16.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base@16.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:78)
	at java.base@16.0.2/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base@16.0.2/java.lang.reflect.Method.invoke(Method.java:567)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:886)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)

[JENKINS] Solr-main-MacOSX (64bit/jdk-17.0.2) - Build # 897 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/897/
Java: 64bit/jdk-17.0.2 -XX:+UseCompressedOops -XX:+UseSerialGC

3 tests failed.
FAILED:  org.apache.solr.cloud.DistribDocExpirationUpdateProcessorTest.testNoAuth

Error Message:
java.lang.AssertionError: Final tally has incorrect numDocsThatNeverExpire expected:<33> but was:<14>

Stack Trace:
java.lang.AssertionError: Final tally has incorrect numDocsThatNeverExpire expected:<33> but was:<14>
	at __randomizedtesting.SeedInfo.seed([65BDA98BB804706E:B90813E602D2A246]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.junit.Assert.failNotEquals(Assert.java:835)
	at org.junit.Assert.assertEquals(Assert.java:647)
	at org.apache.solr.cloud.DistribDocExpirationUpdateProcessorTest.runTest(DistribDocExpirationUpdateProcessorTest.java:289)
	at org.apache.solr.cloud.DistribDocExpirationUpdateProcessorTest.testNoAuth(DistribDocExpirationUpdateProcessorTest.java:134)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:568)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base/java.lang.Thread.run(Thread.java:833)


FAILED:  org.apache.solr.cloud.TestTlogReplica.testRebalanceLeaders

Error Message:
java.lang.AssertionError: Replica core_node3 not up to date after 10 seconds expected:<4> but was:<0>

Stack Trace:
java.lang.AssertionError: Replica core_node3 not up to date after 10 seconds expected:<4> but was:<0>
	at __randomizedtesting.SeedInfo.seed([65BDA98BB804706E:7B529B7B80AF3C1E]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.junit.Assert.failNotEquals(Assert.java:835)
	at org.junit.Assert.assertEquals(Assert.java:647)
	at org.apache.solr.cloud.TestTlogReplica.waitForNumDocsInAllReplicas(TestTlogReplica.java:995)
	at org.apache.solr.cloud.TestTlogReplica.waitForNumDocsInAllReplicas(TestTlogReplica.java:981)
	at org.apache.solr.cloud.TestTlogReplica.waitForNumDocsInAllActiveReplicas(TestTlogReplica.java:971)
	at org.apache.solr.cloud.TestTlogReplica.waitForNumDocsInAllActiveReplicas(TestTlogReplica.java:965)
	at org.apache.solr.cloud.TestTlogReplica.testRebalanceLeaders(TestTlogReplica.java:841)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:568)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base/java.lang.Thread.run(Thread.java:833)


FAILED:  org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=1178, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@17.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:328)
        at app//org.apache.solr.cloud.ZkController.pathExists(ZkController.java:1228)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:73)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getLines(AbstractAnalysisFactory.java:302)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getWordSet(AbstractAnalysisFactory.java:293)
        at app//org.apache.lucene.analysis.en.AbstractWordsFileFilterFactory.inform(AbstractWordsFileFilterFactory.java:88)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:762)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1470)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1319)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1307)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:654)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:188)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:119)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:279)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:342)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:253)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$633/0x00000008011525a0.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:49)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:249)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1850)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3394)
        at app//org.apache.solr.core.SolrCore$$Lambda$743/0x00000008012146d8.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2761)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1204/0x000000080164d160.run(Unknown Source)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=1178, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@17.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:328)
        at app//org.apache.solr.cloud.ZkController.pathExists(ZkController.java:1228)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:73)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getLines(AbstractAnalysisFactory.java:302)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getWordSet(AbstractAnalysisFactory.java:293)
        at app//org.apache.lucene.analysis.en.AbstractWordsFileFilterFactory.inform(AbstractWordsFileFilterFactory.java:88)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:762)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1470)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1319)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1307)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:654)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:188)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:119)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:279)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:342)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:253)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$633/0x00000008011525a0.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:49)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:249)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1850)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3394)
        at app//org.apache.solr.core.SolrCore$$Lambda$743/0x00000008012146d8.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2761)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1204/0x000000080164d160.run(Unknown Source)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
	at __randomizedtesting.SeedInfo.seed([65BDA98BB804706E]:0)

[JENKINS] Solr-main-MacOSX (64bit/jdk-11.0.13) - Build # 896 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/896/
Java: 64bit/jdk-11.0.13 -XX:+UseCompressedOops -XX:+UseSerialGC

2 tests failed.
FAILED:  org.apache.solr.cloud.HttpPartitionOnCommitTest.test

Error Message:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:63074: Underlying core creation failed while creating collection: c8n_2x2_commits

Stack Trace:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:63074: Underlying core creation failed while creating collection: c8n_2x2_commits
	at __randomizedtesting.SeedInfo.seed([A10C9CA0C9CB154B:2958A37A673778B3]:0)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:715)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at app//org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1193)
	at app//org.apache.solr.cloud.AbstractBasicDistributedZkTestBase.createCollection(AbstractBasicDistributedZkTestBase.java:1177)
	at app//org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:2025)
	at app//org.apache.solr.cloud.HttpPartitionOnCommitTest.multiShardTest(HttpPartitionOnCommitTest.java:78)
	at app//org.apache.solr.cloud.HttpPartitionOnCommitTest.test(HttpPartitionOnCommitTest.java:69)
	at java.base@11.0.13/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base@11.0.13/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base@11.0.13/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base@11.0.13/java.lang.reflect.Method.invoke(Method.java:566)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1164)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1135)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)


FAILED:  org.apache.solr.cloud.ReindexCollectionTest.testAbort

Error Message:
java.lang.AssertionError: {responseHeader={status=0, QTime=173}, status={state=running, msg=found [eb9a14cb-2575-4cbe-abcd-f34c913ef9d1] in running tasks}}

Stack Trace:
java.lang.AssertionError: {responseHeader={status=0, QTime=173}, status={state=running, msg=found [eb9a14cb-2575-4cbe-abcd-f34c913ef9d1] in running tasks}}
	at __randomizedtesting.SeedInfo.seed([A10C9CA0C9CB154B:C76F2B37BD118732]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.junit.Assert.assertTrue(Assert.java:42)
	at org.junit.Assert.assertNotNull(Assert.java:713)
	at org.apache.solr.cloud.ReindexCollectionTest.testAbort(ReindexCollectionTest.java:439)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base/java.lang.Thread.run(Thread.java:829)

[JENKINS] Solr-main-MacOSX (64bit/jdk-17.0.2) - Build # 895 - Failure!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/895/
Java: 64bit/jdk-17.0.2 -XX:-UseCompressedOops -XX:+UseParallelGC

No tests ran.

[JENKINS] Solr-main-MacOSX (64bit/jdk-17.0.2) - Build # 894 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/894/
Java: 64bit/jdk-17.0.2 -XX:-UseCompressedOops -XX:+UseG1GC

6 tests failed.
FAILED:  org.apache.solr.cloud.SolrCloudExampleTest.testLoadDocsIntoGettingStartedCollection

Error Message:
java.lang.AssertionError: Collection 'gettingstarted' doesn't exist after trying to create it!

Stack Trace:
java.lang.AssertionError: Collection 'gettingstarted' doesn't exist after trying to create it!
	at __randomizedtesting.SeedInfo.seed([4283632015C87821:51E0514F24A7C187]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.junit.Assert.assertTrue(Assert.java:42)
	at org.apache.solr.cloud.SolrCloudExampleTest.testLoadDocsIntoGettingStartedCollection(SolrCloudExampleTest.java:117)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:568)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1164)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1135)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base/java.lang.Thread.run(Thread.java:833)


FAILED:  org.apache.solr.cloud.TestQueryingOnDownCollection.testQueryToDownCollectionShouldFailFast

Error Message:
junit.framework.AssertionFailedError: Request should fail after trying all replica nodes once

Stack Trace:
junit.framework.AssertionFailedError: Request should fail after trying all replica nodes once
	at __randomizedtesting.SeedInfo.seed([4283632015C87821:915E4A8AB2F9115D]:0)
	at org.apache.lucene.util.LuceneTestCase.expectThrows(LuceneTestCase.java:2874)
	at org.apache.solr.cloud.TestQueryingOnDownCollection.testQueryToDownCollectionShouldFailFast(TestQueryingOnDownCollection.java:96)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:568)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base/java.lang.Thread.run(Thread.java:833)


FAILED:  org.apache.solr.cloud.api.collections.SplitByPrefixTest.doTest

Error Message:
java.lang.AssertionError: Timed out waiting for sub shards to be active.
Timeout waiting to see state for collection=c1 :DocCollection(c1//collections/c1/state.json/14 )={
  "pullReplicas":"0",
  "configName":"conf",
  "replicationFactor":"1",
  "shards":{
    "shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{"core_node2":{
          "core":"c1_shard1_replica_n1",
          "leader":"true",
          "core_node_name":"core_node2",
          "node_name":"127.0.0.1:56080_solr",
          "base_url":"http://127.0.0.1:56080/solr",
          "state":"active",
          "collection":"c1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"}}},
    "shard1_0":{
      "parent":"shard1",
      "shard_parent_node":"127.0.0.1:56080_solr",
      "replicas":{"core_node5":{
          "core":"c1_shard1_0_replica_n3",
          "leader":"true",
          "core_node_name":"core_node5",
          "node_name":"127.0.0.1:56080_solr",
          "base_url":"http://127.0.0.1:56080/solr",
          "state":"active",
          "collection":"c1",
          "shard":"shard1_0",
          "type":"NRT",
          "force_set_state":"false"}},
      "range":"80000000-ffffffff",
      "state":"construction",
      "shard_parent_zk_session":"72174674861031428"},
    "shard1_1":{
      "parent":"shard1",
      "shard_parent_node":"127.0.0.1:56080_solr",
      "replicas":{"core_node6":{
          "core":"c1_shard1_1_replica_n4",
          "leader":"true",
          "core_node_name":"core_node6",
          "node_name":"127.0.0.1:56080_solr",
          "base_url":"http://127.0.0.1:56080/solr",
          "state":"active",
          "collection":"c1",
          "shard":"shard1_1",
          "type":"NRT",
          "force_set_state":"false"}},
      "range":"0-7fffffff",
      "state":"construction",
      "shard_parent_zk_session":"72174674861031428"}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"1",
  "tlogReplicas":"0"}
Live Nodes: [127.0.0.1:56080_solr]
Last available state: DocCollection(c1//collections/c1/state.json/14 )={
  "pullReplicas":"0",
  "configName":"conf",
  "replicationFactor":"1",
  "shards":{
    "shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{"core_node2":{
          "core":"c1_shard1_replica_n1",
          "leader":"true",
          "core_node_name":"core_node2",
          "node_name":"127.0.0.1:56080_solr",
          "base_url":"http://127.0.0.1:56080/solr",
          "state":"active",
          "collection":"c1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"}}},
    "shard1_0":{
      "parent":"shard1",
      "shard_parent_node":"127.0.0.1:56080_solr",
      "replicas":{"core_node5":{
          "core":"c1_shard1_0_replica_n3",
          "leader":"true",
          "core_node_name":"core_node5",
          "node_name":"127.0.0.1:56080_solr",
          "base_url":"http://127.0.0.1:56080/solr",
          "state":"active",
          "collection":"c1",
          "shard":"shard1_0",
          "type":"NRT",
          "force_set_state":"false"}},
      "range":"80000000-ffffffff",
      "state":"construction",
      "shard_parent_zk_session":"72174674861031428"},
    "shard1_1":{
      "parent":"shard1",
      "shard_parent_node":"127.0.0.1:56080_solr",
      "replicas":{"core_node6":{
          "core":"c1_shard1_1_replica_n4",
          "leader":"true",
          "core_node_name":"core_node6",
          "node_name":"127.0.0.1:56080_solr",
          "base_url":"http://127.0.0.1:56080/solr",
          "state":"active",
          "collection":"c1",
          "shard":"shard1_1",
          "type":"NRT",
          "force_set_state":"false"}},
      "range":"0-7fffffff",
      "state":"construction",
      "shard_parent_zk_session":"72174674861031428"}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"1",
  "tlogReplicas":"0"}

Stack Trace:
java.lang.AssertionError: Timed out waiting for sub shards to be active.
Timeout waiting to see state for collection=c1 :DocCollection(c1//collections/c1/state.json/14 )={
  "pullReplicas":"0",
  "configName":"conf",
  "replicationFactor":"1",
  "shards":{
    "shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{"core_node2":{
          "core":"c1_shard1_replica_n1",
          "leader":"true",
          "core_node_name":"core_node2",
          "node_name":"127.0.0.1:56080_solr",
          "base_url":"http://127.0.0.1:56080/solr",
          "state":"active",
          "collection":"c1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"}}},
    "shard1_0":{
      "parent":"shard1",
      "shard_parent_node":"127.0.0.1:56080_solr",
      "replicas":{"core_node5":{
          "core":"c1_shard1_0_replica_n3",
          "leader":"true",
          "core_node_name":"core_node5",
          "node_name":"127.0.0.1:56080_solr",
          "base_url":"http://127.0.0.1:56080/solr",
          "state":"active",
          "collection":"c1",
          "shard":"shard1_0",
          "type":"NRT",
          "force_set_state":"false"}},
      "range":"80000000-ffffffff",
      "state":"construction",
      "shard_parent_zk_session":"72174674861031428"},
    "shard1_1":{
      "parent":"shard1",
      "shard_parent_node":"127.0.0.1:56080_solr",
      "replicas":{"core_node6":{
          "core":"c1_shard1_1_replica_n4",
          "leader":"true",
          "core_node_name":"core_node6",
          "node_name":"127.0.0.1:56080_solr",
          "base_url":"http://127.0.0.1:56080/solr",
          "state":"active",
          "collection":"c1",
          "shard":"shard1_1",
          "type":"NRT",
          "force_set_state":"false"}},
      "range":"0-7fffffff",
      "state":"construction",
      "shard_parent_zk_session":"72174674861031428"}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"1",
  "tlogReplicas":"0"}
Live Nodes: [127.0.0.1:56080_solr]
Last available state: DocCollection(c1//collections/c1/state.json/14 )={
  "pullReplicas":"0",
  "configName":"conf",
  "replicationFactor":"1",
  "shards":{
    "shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{"core_node2":{
          "core":"c1_shard1_replica_n1",
          "leader":"true",
          "core_node_name":"core_node2",
          "node_name":"127.0.0.1:56080_solr",
          "base_url":"http://127.0.0.1:56080/solr",
          "state":"active",
          "collection":"c1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"}}},
    "shard1_0":{
      "parent":"shard1",
      "shard_parent_node":"127.0.0.1:56080_solr",
      "replicas":{"core_node5":{
          "core":"c1_shard1_0_replica_n3",
          "leader":"true",
          "core_node_name":"core_node5",
          "node_name":"127.0.0.1:56080_solr",
          "base_url":"http://127.0.0.1:56080/solr",
          "state":"active",
          "collection":"c1",
          "shard":"shard1_0",
          "type":"NRT",
          "force_set_state":"false"}},
      "range":"80000000-ffffffff",
      "state":"construction",
      "shard_parent_zk_session":"72174674861031428"},
    "shard1_1":{
      "parent":"shard1",
      "shard_parent_node":"127.0.0.1:56080_solr",
      "replicas":{"core_node6":{
          "core":"c1_shard1_1_replica_n4",
          "leader":"true",
          "core_node_name":"core_node6",
          "node_name":"127.0.0.1:56080_solr",
          "base_url":"http://127.0.0.1:56080/solr",
          "state":"active",
          "collection":"c1",
          "shard":"shard1_1",
          "type":"NRT",
          "force_set_state":"false"}},
      "range":"0-7fffffff",
      "state":"construction",
      "shard_parent_zk_session":"72174674861031428"}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"1",
  "tlogReplicas":"0"}
	at __randomizedtesting.SeedInfo.seed([4283632015C87821:E5C7DB8478736B98]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.solr.cloud.SolrCloudTestCase.waitForState(SolrCloudTestCase.java:169)
	at org.apache.solr.cloud.SolrCloudTestCase.waitForState(SolrCloudTestCase.java:135)
	at org.apache.solr.cloud.api.collections.SplitByPrefixTest.doTest(SplitByPrefixTest.java:166)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:568)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base/java.lang.Thread.run(Thread.java:833)


FAILED:  org.apache.solr.handler.designer.TestSchemaDesignerConfigSetHelper.testAnalyzeField

Error Message:
java.lang.Exception: Test abandoned because suite timeout was reached.

Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
	at __randomizedtesting.SeedInfo.seed([4283632015C87821]:0)


FAILED:  org.apache.solr.handler.designer.TestSchemaDesignerConfigSetHelper.classMethod

Error Message:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).

Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
	at __randomizedtesting.SeedInfo.seed([4283632015C87821]:0)


FAILED:  org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=1093, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@17.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:361)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:75)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getLines(AbstractAnalysisFactory.java:302)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getWordSet(AbstractAnalysisFactory.java:293)
        at app//org.apache.lucene.analysis.en.AbstractWordsFileFilterFactory.inform(AbstractWordsFileFilterFactory.java:88)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:762)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1470)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1328)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1307)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:654)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:188)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:119)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:279)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:342)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:253)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$658/0x00000008011dec50.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:49)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:249)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1850)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3394)
        at app//org.apache.solr.core.SolrCore$$Lambda$769/0x00000008012a28d8.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2761)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1215/0x000000080168b078.run(Unknown Source)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=1093, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@17.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:361)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:75)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getLines(AbstractAnalysisFactory.java:302)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getWordSet(AbstractAnalysisFactory.java:293)
        at app//org.apache.lucene.analysis.en.AbstractWordsFileFilterFactory.inform(AbstractWordsFileFilterFactory.java:88)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:762)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1470)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1328)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1307)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:654)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:188)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:119)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:279)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:342)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:253)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$658/0x00000008011dec50.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:49)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:249)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1850)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3394)
        at app//org.apache.solr.core.SolrCore$$Lambda$769/0x00000008012a28d8.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2761)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1215/0x000000080168b078.run(Unknown Source)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
	at __randomizedtesting.SeedInfo.seed([4283632015C87821]:0)

[JENKINS] Solr-main-MacOSX (64bit/jdk-18) - Build # 893 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/893/
Java: 64bit/jdk-18 -XX:-UseCompressedOops -XX:+UseG1GC

3 tests failed.
FAILED:  org.apache.solr.cloud.TestPrepRecovery.testLeaderUnloaded

Error Message:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:60587/solr: Underlying core creation failed while creating collection: testLeaderUnloaded

Stack Trace:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:60587/solr: Underlying core creation failed while creating collection: testLeaderUnloaded
	at __randomizedtesting.SeedInfo.seed([DE9B93CA6D75B373:8D3C803C90595D71]:0)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:715)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:405)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:371)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1248)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:969)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:896)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:223)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:238)
	at app//org.apache.solr.cloud.TestPrepRecovery.testLeaderUnloaded(TestPrepRecovery.java:59)
	at java.base@18/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
	at java.base@18/java.lang.reflect.Method.invoke(Method.java:577)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base@18/java.lang.Thread.run(Thread.java:833)


FAILED:  org.apache.solr.cloud.TestStressLiveNodes.testStress

Error Message:
java.lang.AssertionError: iter125 expected:<19> but was:<21>

Stack Trace:
java.lang.AssertionError: iter125 expected:<19> but was:<21>
	at __randomizedtesting.SeedInfo.seed([DE9B93CA6D75B373:CB80EE0BEE824B08]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.junit.Assert.failNotEquals(Assert.java:835)
	at org.junit.Assert.assertEquals(Assert.java:647)
	at org.apache.solr.cloud.TestStressLiveNodes.testStress(TestStressLiveNodes.java:211)
	at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
	at java.base/java.lang.reflect.Method.invoke(Method.java:577)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base/java.lang.Thread.run(Thread.java:833)


FAILED:  org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=1089, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@18/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:361)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:75)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getLines(AbstractAnalysisFactory.java:302)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getWordSet(AbstractAnalysisFactory.java:293)
        at app//org.apache.lucene.analysis.en.AbstractWordsFileFilterFactory.inform(AbstractWordsFileFilterFactory.java:88)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:762)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1470)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1319)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1307)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:654)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:188)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:119)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:279)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:342)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:253)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$632/0x0000000801137758.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:49)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:249)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1850)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3394)
        at app//org.apache.solr.core.SolrCore$$Lambda$742/0x00000008011f2560.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2761)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1153/0x00000008014e8938.run(Unknown Source)
        at java.base@18/java.lang.Thread.run(Thread.java:833)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=1089, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@18/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:361)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:75)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getLines(AbstractAnalysisFactory.java:302)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getWordSet(AbstractAnalysisFactory.java:293)
        at app//org.apache.lucene.analysis.en.AbstractWordsFileFilterFactory.inform(AbstractWordsFileFilterFactory.java:88)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:762)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1470)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1319)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1307)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:654)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:188)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:119)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:279)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:342)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:253)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$632/0x0000000801137758.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:49)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:249)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1850)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3394)
        at app//org.apache.solr.core.SolrCore$$Lambda$742/0x00000008011f2560.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2761)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1153/0x00000008014e8938.run(Unknown Source)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
	at __randomizedtesting.SeedInfo.seed([DE9B93CA6D75B373]:0)

[JENKINS] Solr-main-MacOSX (64bit/jdk-17.0.2) - Build # 892 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/892/
Java: 64bit/jdk-17.0.2 -XX:+UseCompressedOops -XX:+UseSerialGC

5 tests failed.
FAILED:  org.apache.solr.cloud.HttpPartitionOnCommitTest.test

Error Message:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:56544: Underlying core creation failed while creating collection: c8n_2x2_commits

Stack Trace:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:56544: Underlying core creation failed while creating collection: c8n_2x2_commits
	at __randomizedtesting.SeedInfo.seed([CDDDEB68E21086A3:4589D4B24CECEB5B]:0)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:715)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at app//org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1193)
	at app//org.apache.solr.cloud.AbstractBasicDistributedZkTestBase.createCollection(AbstractBasicDistributedZkTestBase.java:1177)
	at app//org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:2025)
	at app//org.apache.solr.cloud.HttpPartitionOnCommitTest.multiShardTest(HttpPartitionOnCommitTest.java:78)
	at app//org.apache.solr.cloud.HttpPartitionOnCommitTest.test(HttpPartitionOnCommitTest.java:69)
	at java.base@17.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base@17.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base@17.0.2/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base@17.0.2/java.lang.reflect.Method.invoke(Method.java:568)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1164)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1135)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)


FAILED:  org.apache.solr.schema.TestManagedSchemaAPI.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=4257, name=qtp1095722348-4257, state=RUNNABLE, group=TGRP-TestManagedSchemaAPI], registration stack trace below.

Stack Trace:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=4257, name=qtp1095722348-4257, state=RUNNABLE, group=TGRP-TestManagedSchemaAPI], registration stack trace below.
	at java.base@17.0.2/java.lang.Thread.getStackTrace(Thread.java:1610)
	at app//com.carrotsearch.randomizedtesting.RandomizedContext.closeAtEnd(RandomizedContext.java:157)
	at app//org.apache.lucene.util.LuceneTestCase.closeAfterSuite(LuceneTestCase.java:798)
	at app//org.apache.lucene.util.LuceneTestCase.wrapDirectory(LuceneTestCase.java:1504)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1368)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1350)
	at app//org.apache.solr.core.MockDirectoryFactory.create(MockDirectoryFactory.java:51)
	at app//org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:368)
	at app//org.apache.solr.core.SolrCore.initSnapshotMetaDataManager(SolrCore.java:606)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1119)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1048)
	at app//org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1562)
	at app//org.apache.solr.core.CoreContainer.create(CoreContainer.java:1428)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:389)
	at app//org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:422)
	at app//org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:205)
	at app//org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:207)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:941)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:893)
	at app//org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:584)
	at app//org.apache.solr.servlet.SolrDispatchFilter.dispatch(SolrDispatchFilter.java:239)
	at app//org.apache.solr.servlet.SolrDispatchFilter.lambda$doFilter$0(SolrDispatchFilter.java:207)
	at app//org.apache.solr.servlet.ServletUtils.traceHttpRequestExecution2(ServletUtils.java:257)
	at app//org.apache.solr.servlet.ServletUtils.rateLimitRequest(ServletUtils.java:227)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:202)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:184)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:187)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1434)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
	at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)
	at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1349)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
	at app//org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:713)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.server.Server.handle(Server.java:516)
	at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:400)
	at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:645)
	at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:392)
	at app//org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:352)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
	at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
Caused by: java.lang.AssertionError: Directory not closed: MockDirectoryWrapper(ByteBuffersDirectory@4aa2709d lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@254f01db)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.lucene.util.CloseableDirectory.close(CloseableDirectory.java:43)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:789)
	at com.carrotsearch.randomizedtesting.RandomizedContext.closeResources(RandomizedContext.java:225)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.afterAlways(RandomizedRunner.java:805)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	... 1 more


FAILED:  org.apache.solr.schema.TestManagedSchemaAPI.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=4257, name=qtp1095722348-4257, state=RUNNABLE, group=TGRP-TestManagedSchemaAPI], registration stack trace below.

Stack Trace:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=4257, name=qtp1095722348-4257, state=RUNNABLE, group=TGRP-TestManagedSchemaAPI], registration stack trace below.
	at java.base@17.0.2/java.lang.Thread.getStackTrace(Thread.java:1610)
	at app//com.carrotsearch.randomizedtesting.RandomizedContext.closeAtEnd(RandomizedContext.java:157)
	at app//org.apache.lucene.util.LuceneTestCase.closeAfterSuite(LuceneTestCase.java:798)
	at app//org.apache.lucene.util.LuceneTestCase.wrapDirectory(LuceneTestCase.java:1504)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1368)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1350)
	at app//org.apache.solr.core.MockDirectoryFactory.create(MockDirectoryFactory.java:51)
	at app//org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:368)
	at app//org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:412)
	at app//org.apache.solr.core.SolrCore.initIndex(SolrCore.java:856)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1123)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1048)
	at app//org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1562)
	at app//org.apache.solr.core.CoreContainer.create(CoreContainer.java:1428)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:389)
	at app//org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:422)
	at app//org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:205)
	at app//org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:207)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:941)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:893)
	at app//org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:584)
	at app//org.apache.solr.servlet.SolrDispatchFilter.dispatch(SolrDispatchFilter.java:239)
	at app//org.apache.solr.servlet.SolrDispatchFilter.lambda$doFilter$0(SolrDispatchFilter.java:207)
	at app//org.apache.solr.servlet.ServletUtils.traceHttpRequestExecution2(ServletUtils.java:257)
	at app//org.apache.solr.servlet.ServletUtils.rateLimitRequest(ServletUtils.java:227)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:202)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:184)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:187)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1434)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
	at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)
	at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1349)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
	at app//org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:713)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.server.Server.handle(Server.java:516)
	at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:400)
	at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:645)
	at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:392)
	at app//org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:352)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
	at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
Caused by: java.lang.AssertionError: Directory not closed: MockDirectoryWrapper(ByteBuffersDirectory@13f4ccbc lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@248ca88b)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.lucene.util.CloseableDirectory.close(CloseableDirectory.java:43)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:789)
	at com.carrotsearch.randomizedtesting.RandomizedContext.closeResources(RandomizedContext.java:225)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.afterAlways(RandomizedRunner.java:805)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	... 1 more


FAILED:  org.apache.solr.schema.TestManagedSchemaAPI.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=4257, name=qtp1095722348-4257, state=RUNNABLE, group=TGRP-TestManagedSchemaAPI], registration stack trace below.

Stack Trace:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=4257, name=qtp1095722348-4257, state=RUNNABLE, group=TGRP-TestManagedSchemaAPI], registration stack trace below.
	at java.base@17.0.2/java.lang.Thread.getStackTrace(Thread.java:1610)
	at app//com.carrotsearch.randomizedtesting.RandomizedContext.closeAtEnd(RandomizedContext.java:157)
	at app//org.apache.lucene.util.LuceneTestCase.closeAfterSuite(LuceneTestCase.java:798)
	at app//org.apache.lucene.util.LuceneTestCase.wrapDirectory(LuceneTestCase.java:1504)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1368)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1350)
	at app//org.apache.solr.core.MockDirectoryFactory.create(MockDirectoryFactory.java:51)
	at app//org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:368)
	at app//org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:108)
	at app//org.apache.solr.core.SolrCore.initIndex(SolrCore.java:895)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1123)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1048)
	at app//org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1562)
	at app//org.apache.solr.core.CoreContainer.create(CoreContainer.java:1428)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:389)
	at app//org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:422)
	at app//org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:205)
	at app//org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:207)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:941)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:893)
	at app//org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:584)
	at app//org.apache.solr.servlet.SolrDispatchFilter.dispatch(SolrDispatchFilter.java:239)
	at app//org.apache.solr.servlet.SolrDispatchFilter.lambda$doFilter$0(SolrDispatchFilter.java:207)
	at app//org.apache.solr.servlet.ServletUtils.traceHttpRequestExecution2(ServletUtils.java:257)
	at app//org.apache.solr.servlet.ServletUtils.rateLimitRequest(ServletUtils.java:227)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:202)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:184)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:187)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1434)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
	at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)
	at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1349)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
	at app//org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:713)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.server.Server.handle(Server.java:516)
	at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:400)
	at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:645)
	at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:392)
	at app//org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:352)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
	at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
Caused by: java.lang.AssertionError: Directory not closed: MockDirectoryWrapper(ByteBuffersDirectory@34bd4d83 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6f67836b)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.lucene.util.CloseableDirectory.close(CloseableDirectory.java:43)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:789)
	at com.carrotsearch.randomizedtesting.RandomizedContext.closeResources(RandomizedContext.java:225)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.afterAlways(RandomizedRunner.java:805)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	... 1 more


FAILED:  org.apache.solr.schema.TestManagedSchemaAPI.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.schema.TestManagedSchemaAPI: 
   1) Thread[id=4358, name=searcherExecutor-3784-thread-1-processing-127.0.0.1:51608_solr, state=WAITING, group=TGRP-TestManagedSchemaAPI]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
        at java.base@17.0.2/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:230)
        at app//org.apache.solr.core.SolrCore.lambda$new$2(SolrCore.java:1142)
        at app//org.apache.solr.core.SolrCore$$Lambda$722/0x0000000801204b10.call(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.FutureTask.run(FutureTask.java:264)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$264/0x0000000800f86000.run(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   2) Thread[id=4351, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestManagedSchemaAPI]
        at java.base@17.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:361)
        at app//org.apache.solr.schema.ZkIndexSchemaReader.updateSchema(ZkIndexSchemaReader.java:176)
        at app//org.apache.solr.schema.ZkIndexSchemaReader.refreshSchemaFromZk(ZkIndexSchemaReader.java:165)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.inform(ManagedIndexSchemaFactory.java:541)
        at app//org.apache.solr.core.SolrResourceLoader.inform(SolrResourceLoader.java:728)
        at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1155)
        at app//org.apache.solr.core.SolrCore.reload(SolrCore.java:780)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1857)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3394)
        at app//org.apache.solr.core.SolrCore$$Lambda$745/0x000000080121a6d0.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2761)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1213/0x0000000801638db0.run(Unknown Source)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.schema.TestManagedSchemaAPI: 
   1) Thread[id=4358, name=searcherExecutor-3784-thread-1-processing-127.0.0.1:51608_solr, state=WAITING, group=TGRP-TestManagedSchemaAPI]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
        at java.base@17.0.2/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:230)
        at app//org.apache.solr.core.SolrCore.lambda$new$2(SolrCore.java:1142)
        at app//org.apache.solr.core.SolrCore$$Lambda$722/0x0000000801204b10.call(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.FutureTask.run(FutureTask.java:264)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$264/0x0000000800f86000.run(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   2) Thread[id=4351, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestManagedSchemaAPI]
        at java.base@17.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:361)
        at app//org.apache.solr.schema.ZkIndexSchemaReader.updateSchema(ZkIndexSchemaReader.java:176)
        at app//org.apache.solr.schema.ZkIndexSchemaReader.refreshSchemaFromZk(ZkIndexSchemaReader.java:165)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.inform(ManagedIndexSchemaFactory.java:541)
        at app//org.apache.solr.core.SolrResourceLoader.inform(SolrResourceLoader.java:728)
        at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1155)
        at app//org.apache.solr.core.SolrCore.reload(SolrCore.java:780)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1857)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3394)
        at app//org.apache.solr.core.SolrCore$$Lambda$745/0x000000080121a6d0.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2761)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1213/0x0000000801638db0.run(Unknown Source)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
	at __randomizedtesting.SeedInfo.seed([CDDDEB68E21086A3]:0)

[JENKINS] Solr-main-MacOSX (64bit/jdk-17.0.2) - Build # 891 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/891/
Java: 64bit/jdk-17.0.2 -XX:+UseCompressedOops -XX:+UseSerialGC

4 tests failed.
FAILED:  org.apache.solr.cloud.HttpPartitionTest.test

Error Message:
java.lang.AssertionError: Didn't see all replicas for shard shard1 in collDoRecoveryOnRestart come up within 130000 ms! ClusterState: {
  "control_collection":{
    "pullReplicas":"0",
    "configName":"conf1",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{"core_node2":{
            "core":"control_collection_shard1_replica_n1",
            "leader":"true",
            "core_node_name":"core_node2",
            "node_name":"127.0.0.1:57994_",
            "base_url":"http://127.0.0.1:57994",
            "state":"active",
            "collection":"control_collection",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"},
  "collDoRecoveryOnRestart":{
    "pullReplicas":"0",
    "configName":"conf1",
    "replicationFactor":"2",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"collDoRecoveryOnRestart_shard1_replica_n1",
            "core_node_name":"core_node3",
            "node_name":"127.0.0.1:58055_",
            "base_url":"http://127.0.0.1:58055",
            "state":"recovering",
            "collection":"collDoRecoveryOnRestart",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node4":{
            "core":"collDoRecoveryOnRestart_shard1_replica_n2",
            "leader":"true",
            "core_node_name":"core_node4",
            "node_name":"127.0.0.1:57994_",
            "base_url":"http://127.0.0.1:57994",
            "state":"active",
            "collection":"collDoRecoveryOnRestart",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"2",
    "tlogReplicas":"0"},
  "collection1":{
    "pullReplicas":"0",
    "configName":"conf1",
    "replicationFactor":"1",
    "shards":{
      "shard1":{
        "range":"80000000-ffffffff",
        "state":"active",
        "replicas":{"core_node4":{
            "core":"collection1_shard1_replica_n2",
            "leader":"true",
            "core_node_name":"core_node4",
            "node_name":"127.0.0.1:58069_",
            "base_url":"http://127.0.0.1:58069",
            "state":"active",
            "collection":"collection1",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}},
      "shard2":{
        "range":"0-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"collection1_shard2_replica_n1",
            "core_node_name":"core_node3",
            "node_name":"127.0.0.1:58055_",
            "base_url":"http://127.0.0.1:58055",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node6":{
            "core":"collection1_shard2_replica_n5",
            "leader":"true",
            "core_node_name":"core_node6",
            "node_name":"127.0.0.1:58086_",
            "base_url":"http://127.0.0.1:58086",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"}}

Stack Trace:
java.lang.AssertionError: Didn't see all replicas for shard shard1 in collDoRecoveryOnRestart come up within 130000 ms! ClusterState: {
  "control_collection":{
    "pullReplicas":"0",
    "configName":"conf1",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{"core_node2":{
            "core":"control_collection_shard1_replica_n1",
            "leader":"true",
            "core_node_name":"core_node2",
            "node_name":"127.0.0.1:57994_",
            "base_url":"http://127.0.0.1:57994",
            "state":"active",
            "collection":"control_collection",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"},
  "collDoRecoveryOnRestart":{
    "pullReplicas":"0",
    "configName":"conf1",
    "replicationFactor":"2",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"collDoRecoveryOnRestart_shard1_replica_n1",
            "core_node_name":"core_node3",
            "node_name":"127.0.0.1:58055_",
            "base_url":"http://127.0.0.1:58055",
            "state":"recovering",
            "collection":"collDoRecoveryOnRestart",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node4":{
            "core":"collDoRecoveryOnRestart_shard1_replica_n2",
            "leader":"true",
            "core_node_name":"core_node4",
            "node_name":"127.0.0.1:57994_",
            "base_url":"http://127.0.0.1:57994",
            "state":"active",
            "collection":"collDoRecoveryOnRestart",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"2",
    "tlogReplicas":"0"},
  "collection1":{
    "pullReplicas":"0",
    "configName":"conf1",
    "replicationFactor":"1",
    "shards":{
      "shard1":{
        "range":"80000000-ffffffff",
        "state":"active",
        "replicas":{"core_node4":{
            "core":"collection1_shard1_replica_n2",
            "leader":"true",
            "core_node_name":"core_node4",
            "node_name":"127.0.0.1:58069_",
            "base_url":"http://127.0.0.1:58069",
            "state":"active",
            "collection":"collection1",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}},
      "shard2":{
        "range":"0-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"collection1_shard2_replica_n1",
            "core_node_name":"core_node3",
            "node_name":"127.0.0.1:58055_",
            "base_url":"http://127.0.0.1:58055",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node6":{
            "core":"collection1_shard2_replica_n5",
            "leader":"true",
            "core_node_name":"core_node6",
            "node_name":"127.0.0.1:58086_",
            "base_url":"http://127.0.0.1:58086",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"}}
	at __randomizedtesting.SeedInfo.seed([F66868A3869D7B1B:7E3C5779286116E3]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.ensureAllReplicasAreActive(AbstractFullDistribZkTestBase.java:2516)
	at org.apache.solr.cloud.HttpPartitionTest.testDoRecoveryOnRestart(HttpPartitionTest.java:207)
	at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:133)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:568)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1164)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1135)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base/java.lang.Thread.run(Thread.java:833)


FAILED:  org.apache.solr.cloud.TestQueryingOnDownCollection.testQueryToDownCollectionShouldFailFast

Error Message:
junit.framework.AssertionFailedError: Request should fail after trying all replica nodes once

Stack Trace:
junit.framework.AssertionFailedError: Request should fail after trying all replica nodes once
	at __randomizedtesting.SeedInfo.seed([F66868A3869D7B1B:25B5410921AC1267]:0)
	at org.apache.lucene.util.LuceneTestCase.expectThrows(LuceneTestCase.java:2874)
	at org.apache.solr.cloud.TestQueryingOnDownCollection.testQueryToDownCollectionShouldFailFast(TestQueryingOnDownCollection.java:96)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:568)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base/java.lang.Thread.run(Thread.java:833)


FAILED:  org.apache.solr.cloud.TestSizeLimitedDistributedMap.testCleanup

Error Message:
org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /org.apache.solr.cloud.TestSizeLimitedDistributedMap/testCleanup/mn-xyz_1

Stack Trace:
org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /org.apache.solr.cloud.TestSizeLimitedDistributedMap/testCleanup/mn-xyz_1
	at __randomizedtesting.SeedInfo.seed([F66868A3869D7B1B:A6423B0DFF30EBAE]:0)
	at app//org.apache.zookeeper.KeeperException.create(KeeperException.java:118)
	at app//org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
	at app//org.apache.zookeeper.ZooKeeper.delete(ZooKeeper.java:1614)
	at app//org.apache.solr.common.cloud.SolrZkClient.lambda$delete$1(SolrZkClient.java:281)
	at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:72)
	at app//org.apache.solr.common.cloud.SolrZkClient.delete(SolrZkClient.java:279)
	at app//org.apache.solr.cloud.SizeLimitedDistributedMap.shrinkIfNeeded(SizeLimitedDistributedMap.java:94)
	at app//org.apache.solr.cloud.SizeLimitedDistributedMap.put(SizeLimitedDistributedMap.java:54)
	at app//org.apache.solr.cloud.TestSizeLimitedDistributedMap.testCleanup(TestSizeLimitedDistributedMap.java:47)
	at java.base@17.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base@17.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base@17.0.2/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base@17.0.2/java.lang.reflect.Method.invoke(Method.java:568)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)


FAILED:  org.apache.solr.cloud.api.collections.TestReplicaProperties.test

Error Message:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:53298: Underlying core creation failed while creating collection: testcollection

Stack Trace:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:53298: Underlying core creation failed while creating collection: testcollection
	at __randomizedtesting.SeedInfo.seed([F66868A3869D7B1B:7E3C5779286116E3]:0)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:715)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:405)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:371)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1248)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:969)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:896)
	at app//org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1193)
	at app//org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:2102)
	at app//org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:2162)
	at app//org.apache.solr.cloud.api.collections.TestReplicaProperties.test(TestReplicaProperties.java:59)
	at java.base@17.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base@17.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base@17.0.2/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base@17.0.2/java.lang.reflect.Method.invoke(Method.java:568)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1164)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1135)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)

[JENKINS] Solr-main-MacOSX (64bit/jdk-17.0.2) - Build # 890 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/890/
Java: 64bit/jdk-17.0.2 -XX:-UseCompressedOops -XX:+UseSerialGC

2 tests failed.
FAILED:  org.apache.solr.update.processor.DocExpirationUpdateProcessorFactoryTest.testAutomaticDeletes

Error Message:
java.lang.NullPointerException: Cannot invoke "Object.getClass()" because "tmp" is null

Stack Trace:
java.lang.NullPointerException: Cannot invoke "Object.getClass()" because "tmp" is null
	at __randomizedtesting.SeedInfo.seed([6859E5D117D9BAB1:A54A9FABAAAFA94D]:0)
	at org.apache.solr.update.processor.DocExpirationUpdateProcessorFactoryTest.testAutomaticDeletes(DocExpirationUpdateProcessorFactoryTest.java:222)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:568)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base/java.lang.Thread.run(Thread.java:833)


FAILED:  org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=1078, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@17.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:361)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:75)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getLines(AbstractAnalysisFactory.java:302)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getWordSet(AbstractAnalysisFactory.java:293)
        at app//org.apache.lucene.analysis.en.AbstractWordsFileFilterFactory.inform(AbstractWordsFileFilterFactory.java:88)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:762)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1470)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1319)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1307)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:654)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:188)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:119)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:279)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:342)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:253)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$658/0x00000008011d88b8.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:49)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:249)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1850)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3394)
        at app//org.apache.solr.core.SolrCore$$Lambda$768/0x000000080129b5d8.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2761)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1209/0x0000000801681210.run(Unknown Source)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=1078, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@17.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:361)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:75)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getLines(AbstractAnalysisFactory.java:302)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getWordSet(AbstractAnalysisFactory.java:293)
        at app//org.apache.lucene.analysis.en.AbstractWordsFileFilterFactory.inform(AbstractWordsFileFilterFactory.java:88)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:762)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1470)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1319)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1307)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:654)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:188)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:119)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:279)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:342)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:253)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$658/0x00000008011d88b8.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:49)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:249)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1850)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3394)
        at app//org.apache.solr.core.SolrCore$$Lambda$768/0x000000080129b5d8.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2761)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1209/0x0000000801681210.run(Unknown Source)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
	at __randomizedtesting.SeedInfo.seed([6859E5D117D9BAB1]:0)

[JENKINS] Solr-main-MacOSX (64bit/jdk-16.0.2) - Build # 889 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/889/
Java: 64bit/jdk-16.0.2 -XX:-UseCompressedOops -XX:+UseParallelGC

9 tests failed.
FAILED:  org.apache.solr.cloud.CreateRoutedAliasTest.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.cloud.CreateRoutedAliasTest: 
   1) Thread[id=18030, name=ConnnectionExpirer, state=TIMED_WAITING, group=TGRP-CreateRoutedAliasTest]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.server.NIOServerCnxnFactory$ConnectionExpirerThread.run(NIOServerCnxnFactory.java:554)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.cloud.CreateRoutedAliasTest: 
   1) Thread[id=18030, name=ConnnectionExpirer, state=TIMED_WAITING, group=TGRP-CreateRoutedAliasTest]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.server.NIOServerCnxnFactory$ConnectionExpirerThread.run(NIOServerCnxnFactory.java:554)
	at __randomizedtesting.SeedInfo.seed([EC1DB1CE26ED2B60]:0)


FAILED:  org.apache.solr.cloud.HttpPartitionOnCommitTest.test

Error Message:
org.apache.solr.cloud.ZkController$NotInClusterStateException: Number of replicas in the state does not match what we set:3 vs 4

Stack Trace:
org.apache.solr.cloud.ZkController$NotInClusterStateException: Number of replicas in the state does not match what we set:3 vs 4
	at __randomizedtesting.SeedInfo.seed([EC1DB1CE26ED2B60:64498E1488114698]:0)
	at app//org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForActiveReplicaCount(AbstractFullDistribZkTestBase.java:709)
	at app//org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:639)
	at app//org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:378)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1162)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1135)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)


FAILED:  org.apache.solr.cloud.HttpPartitionOnCommitTest.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 105 threads leaked from SUITE scope at org.apache.solr.cloud.HttpPartitionOnCommitTest: 
   1) Thread[id=20126, name=watches-13017-thread-3, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   2) Thread[id=19981, name=qtp847956012-19981, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   3) Thread[id=20027, name=zkConnectionManagerCallback-13016-thread-1, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   4) Thread[id=20006, name=qtp1593956388-20006, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   5) Thread[id=20038, name=qtp1024122880-20038, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   6) Thread[id=20031, name=SocketProxy-Acceptor-51041, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.Net.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.park(NioSocketImpl.java:181)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.timedAccept(NioSocketImpl.java:710)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.accept(NioSocketImpl.java:752)
        at java.base@16.0.2/java.net.ServerSocket.implAccept(ServerSocket.java:681)
        at java.base@16.0.2/java.net.ServerSocket.platformImplAccept(ServerSocket.java:647)
        at java.base@16.0.2/java.net.ServerSocket.implAccept(ServerSocket.java:623)
        at java.base@16.0.2/java.net.ServerSocket.implAccept(ServerSocket.java:580)
        at java.base@16.0.2/java.net.ServerSocket.accept(ServerSocket.java:538)
        at app//org.apache.solr.client.solrj.cloud.SocketProxy$Acceptor.run(SocketProxy.java:445)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   7) Thread[id=20155, name=zkCallback-13015-thread-5, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   8) Thread[id=19950, name=qtp2035113691-19950, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   9) Thread[id=19980, name=qtp847956012-19980-acceptor-0@449afce5-ServerConnector@186620a7{HTTP/1.1, (http/1.1, h2c)}{127.0.0.1:51028}, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.Net.accept(Native Method)
        at java.base@16.0.2/sun.nio.ch.ServerSocketChannelImpl.implAccept(ServerSocketChannelImpl.java:424)
        at java.base@16.0.2/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:390)
        at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388)
        at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:702)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  10) Thread[id=20014, name=Session-HouseKeeper-7b5e93a0-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  11) Thread[id=20018, name=httpShardExecutor-13009-thread-1, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$267/0x0000000800f8c7a0.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  12) Thread[id=19955, name=qtp2035113691-19955, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  13) Thread[id=20152, name=zkCallback-12977-thread-4, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  14) Thread[id=20128, name=h2sc-12973-thread-6, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  15) Thread[id=19978, name=qtp847956012-19978, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  16) Thread[id=20000, name=CloserThread, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/java.lang.Object.wait(Native Method)
        at java.base@16.0.2/java.lang.Object.wait(Object.java:320)
        at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2409)
  17) Thread[id=20088, name=watches-12998-thread-2-processing-127.0.0.1:51027_kvc%2Fm collection1_shard1_replica_n3 collection1 shard1 core_node4, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkStateReader$CacheCleaner.run(ZkStateReader.java:2332)
        at java.base@16.0.2/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
        at java.base@16.0.2/java.util.concurrent.FutureTask.run(FutureTask.java:264)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$267/0x0000000800f8c7a0.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  18) Thread[id=20030, name=coreContainerWorkExecutor-13006-thread-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  19) Thread[id=20040, name=qtp1024122880-20040, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  20) Thread[id=20113, name=updateExecutor-13010-thread-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  21) Thread[id=19951, name=qtp2035113691-19951-acceptor-0@7b549fb1-ServerConnector@7d72d52a{HTTP/1.1, (http/1.1, h2c)}{127.0.0.1:51013}, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.Net.accept(Native Method)
        at java.base@16.0.2/sun.nio.ch.ServerSocketChannelImpl.implAccept(ServerSocketChannelImpl.java:424)
        at java.base@16.0.2/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:390)
        at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388)
        at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:702)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  22) Thread[id=19982, name=qtp847956012-19982, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  23) Thread[id=20058, name=coreContainerWorkExecutor-13025-thread-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  24) Thread[id=20009, name=qtp1593956388-20009-acceptor-0@7d6c2e71-ServerConnector@27960ae1{HTTP/1.1, (http/1.1, h2c)}{127.0.0.1:51043}, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.Net.accept(Native Method)
        at java.base@16.0.2/sun.nio.ch.ServerSocketChannelImpl.implAccept(ServerSocketChannelImpl.java:424)
        at java.base@16.0.2/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:390)
        at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388)
        at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:702)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  25) Thread[id=20010, name=qtp1593956388-20010, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  26) Thread[id=19952, name=qtp2035113691-19952, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  27) Thread[id=20001, name=coreContainerWorkExecutor-12987-thread-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  28) Thread[id=19953, name=qtp2035113691-19953, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  29) Thread[id=19983, name=qtp847956012-19983, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  30) Thread[id=20153, name=zkCallback-13015-thread-4, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  31) Thread[id=19999, name=zkCallback-12996-thread-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  32) Thread[id=20039, name=qtp1024122880-20039, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  33) Thread[id=20098, name=updateExecutor-12972-thread-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  34) Thread[id=20116, name=SocketProxy-Response-51013:51097, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.Net.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.park(NioSocketImpl.java:181)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.timedRead(NioSocketImpl.java:285)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.implRead(NioSocketImpl.java:309)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.read(NioSocketImpl.java:350)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl$1.read(NioSocketImpl.java:803)
        at java.base@16.0.2/java.net.Socket$SocketInputStream.read(Socket.java:976)
        at java.base@16.0.2/java.io.InputStream.read(InputStream.java:218)
        at app//org.apache.solr.client.solrj.cloud.SocketProxy$Bridge$Pump.run(SocketProxy.java:375)
  35) Thread[id=19998, name=zkConnectionManagerCallback-12997-thread-1, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  36) Thread[id=19977, name=qtp847956012-19977, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  37) Thread[id=20093, name=recoveryExecutor-12993-thread-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  38) Thread[id=20029, name=CloserThread, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/java.lang.Object.wait(Native Method)
        at java.base@16.0.2/java.lang.Object.wait(Object.java:320)
        at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2409)
  39) Thread[id=19972, name=CloserThread, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/java.lang.Object.wait(Native Method)
        at java.base@16.0.2/java.lang.Object.wait(Object.java:320)
        at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2409)
  40) Thread[id=20158, name=zkCallback-12996-thread-4, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  41) Thread[id=20159, name=zkCallback-13034-thread-3, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  42) Thread[id=20101, name=h2sc-12973-thread-3, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  43) Thread[id=19954, name=qtp2035113691-19954, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  44) Thread[id=20041, name=qtp1024122880-20041, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  45) Thread[id=20160, name=zkCallback-12996-thread-5, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  46) Thread[id=20055, name=zkConnectionManagerCallback-13035-thread-1, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  47) Thread[id=20115, name=SocketProxy-Request-51097:51013, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.Net.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.park(NioSocketImpl.java:181)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.timedRead(NioSocketImpl.java:285)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.implRead(NioSocketImpl.java:309)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.read(NioSocketImpl.java:350)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl$1.read(NioSocketImpl.java:803)
        at java.base@16.0.2/java.net.Socket$SocketInputStream.read(Socket.java:976)
        at java.base@16.0.2/java.io.InputStream.read(InputStream.java:218)
        at app//org.apache.solr.client.solrj.cloud.SocketProxy$Bridge$Pump.run(SocketProxy.java:375)
  48) Thread[id=20104, name=watches-12979-thread-3, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  49) Thread[id=20036, name=qtp1024122880-20036, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  50) Thread[id=20081, name=zkCallback-12977-thread-3, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  51) Thread[id=19979, name=qtp847956012-19979, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  52) Thread[id=20121, name=h2sc-12973-thread-5, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  53) Thread[id=20157, name=zkCallback-13034-thread-2, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  54) Thread[id=20114, name=recoveryExecutor-13012-thread-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  55) Thread[id=20118, name=zkCallback-13015-thread-3, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  56) Thread[id=20125, name=watches-12998-thread-3, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  57) Thread[id=19985, name=Session-HouseKeeper-5d93b66e-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  58) Thread[id=20078, name=zkCallback-12977-thread-2, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  59) Thread[id=20156, name=zkCallback-12977-thread-6, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  60) Thread[id=20011, name=qtp1593956388-20011, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  61) Thread[id=20034, name=qtp1024122880-20034, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  62) Thread[id=20049, name=h2sc-13030-thread-1, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$267/0x0000000800f8c7a0.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  63) Thread[id=20013, name=qtp1593956388-20013, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  64) Thread[id=20095, name=SocketProxy-Response-51013:51082, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.Net.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.park(NioSocketImpl.java:181)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.timedRead(NioSocketImpl.java:285)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.implRead(NioSocketImpl.java:309)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.read(NioSocketImpl.java:350)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl$1.read(NioSocketImpl.java:803)
        at java.base@16.0.2/java.net.Socket$SocketInputStream.read(Socket.java:976)
        at java.base@16.0.2/java.io.InputStream.read(InputStream.java:218)
        at app//org.apache.solr.client.solrj.cloud.SocketProxy$Bridge$Pump.run(SocketProxy.java:375)
  65) Thread[id=20035, name=qtp1024122880-20035, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  66) Thread[id=20037, name=qtp1024122880-20037-acceptor-0@2be775ce-ServerConnector@7a90fc7b{HTTP/1.1, (http/1.1, h2c)}{127.0.0.1:51053}, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.Net.accept(Native Method)
        at java.base@16.0.2/sun.nio.ch.ServerSocketChannelImpl.implAccept(ServerSocketChannelImpl.java:424)
        at java.base@16.0.2/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:390)
        at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388)
        at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:702)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  67) Thread[id=20089, name=zkCallback-12996-thread-2, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  68) Thread[id=20046, name=httpShardExecutor-13028-thread-1, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$267/0x0000000800f8c7a0.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  69) Thread[id=19963, name=h2sc-12973-thread-1, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$267/0x0000000800f8c7a0.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  70) Thread[id=20094, name=SocketProxy-Request-51082:51013, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.Net.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.park(NioSocketImpl.java:181)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.timedRead(NioSocketImpl.java:285)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.implRead(NioSocketImpl.java:309)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.read(NioSocketImpl.java:350)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl$1.read(NioSocketImpl.java:803)
        at java.base@16.0.2/java.net.Socket$SocketInputStream.read(Socket.java:976)
        at java.base@16.0.2/java.io.InputStream.read(InputStream.java:218)
        at app//org.apache.solr.client.solrj.cloud.SocketProxy$Bridge$Pump.run(SocketProxy.java:375)
  71) Thread[id=20120, name=h2sc-12973-thread-4, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  72) Thread[id=20042, name=Session-HouseKeeper-1ed1a01d-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  73) Thread[id=20124, name=watches-13017-thread-2, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  74) Thread[id=20150, name=OverseerStateUpdate-72169010312052745-127.0.0.1:51012_kvc%2Fm-n_0000000001, state=TIMED_WAITING, group=Overseer state updater.]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:488)
        at app//org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:320)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  75) Thread[id=19992, name=h2sc-12992-thread-1, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$267/0x0000000800f8c7a0.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  76) Thread[id=20056, name=zkCallback-13034-thread-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  77) Thread[id=20154, name=zkCallback-12977-thread-5, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  78) Thread[id=20021, name=h2sc-13011-thread-1, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$267/0x0000000800f8c7a0.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  79) Thread[id=20112, name=searcherExecutor-13056-thread-1, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  80) Thread[id=20087, name=watches-12998-thread-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  81) Thread[id=19984, name=qtp847956012-19984, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  82) Thread[id=20075, name=Connector-Scheduler-7d72d52a-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  83) Thread[id=20091, name=searcherExecutor-13049-thread-1, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  84) Thread[id=20107, name=Connector-Scheduler-27960ae1-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  85) Thread[id=19973, name=coreContainerWorkExecutor-12968-thread-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  86) Thread[id=20083, name=watches-12979-thread-2, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  87) Thread[id=19970, name=zkCallback-12977-thread-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  88) Thread[id=20109, name=watches-13017-thread-1-processing-127.0.0.1:51041_kvc%2Fm collection1_shard1_replica_n5 collection1 shard1 core_node6, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkStateReader$CacheCleaner.run(ZkStateReader.java:2332)
        at java.base@16.0.2/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
        at java.base@16.0.2/java.util.concurrent.FutureTask.run(FutureTask.java:264)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$267/0x0000000800f8c7a0.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  89) Thread[id=20099, name=HttpClient@59ad64a7-scheduler-1, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  90) Thread[id=20100, name=h2sc-12973-thread-2, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  91) Thread[id=20012, name=qtp1593956388-20012, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  92) Thread[id=20119, name=updateExecutor-12972-thread-2, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  93) Thread[id=19989, name=httpShardExecutor-12990-thread-1, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$267/0x0000000800f8c7a0.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  94) Thread[id=19974, name=SocketProxy-Acceptor-51012, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.Net.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.park(NioSocketImpl.java:181)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.timedAccept(NioSocketImpl.java:710)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.accept(NioSocketImpl.java:752)
        at java.base@16.0.2/java.net.ServerSocket.implAccept(ServerSocket.java:681)
        at java.base@16.0.2/java.net.ServerSocket.platformImplAccept(ServerSocket.java:647)
        at java.base@16.0.2/java.net.ServerSocket.implAccept(ServerSocket.java:623)
        at java.base@16.0.2/java.net.ServerSocket.implAccept(ServerSocket.java:580)
        at java.base@16.0.2/java.net.ServerSocket.accept(ServerSocket.java:538)
        at app//org.apache.solr.client.solrj.cloud.SocketProxy$Acceptor.run(SocketProxy.java:445)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  95) Thread[id=20097, name=zkCallback-12996-thread-3, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  96) Thread[id=20161, name=zkCallback-13034-thread-4, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  97) Thread[id=20151, name=OverseerCollectionConfigSetProcessor-72169010312052745-127.0.0.1:51012_kvc%2Fm-n_0000000001, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:488)
        at app//org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:266)
        at app//org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:274)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  98) Thread[id=19960, name=httpShardExecutor-12971-thread-1, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$267/0x0000000800f8c7a0.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  99) Thread[id=20002, name=SocketProxy-Acceptor-51027, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.Net.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.park(NioSocketImpl.java:181)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.timedAccept(NioSocketImpl.java:710)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.accept(NioSocketImpl.java:752)
        at java.base@16.0.2/java.net.ServerSocket.implAccept(ServerSocket.java:681)
        at java.base@16.0.2/java.net.ServerSocket.platformImplAccept(ServerSocket.java:647)
        at java.base@16.0.2/java.net.ServerSocket.implAccept(ServerSocket.java:623)
        at java.base@16.0.2/java.net.ServerSocket.implAccept(ServerSocket.java:580)
        at java.base@16.0.2/java.net.ServerSocket.accept(ServerSocket.java:538)
        at app//org.apache.solr.client.solrj.cloud.SocketProxy$Acceptor.run(SocketProxy.java:445)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  100) Thread[id=20008, name=qtp1593956388-20008, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  101) Thread[id=20129, name=h2sc-12973-thread-7, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  102) Thread[id=19969, name=zkConnectionManagerCallback-12978-thread-1, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  103) Thread[id=20092, name=updateExecutor-12991-thread-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  104) Thread[id=19956, name=Session-HouseKeeper-3e0774b4-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  105) Thread[id=20028, name=zkCallback-13015-thread-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  106) Thread[id=20059, name=SocketProxy-Acceptor-51052, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.Net.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.park(NioSocketImpl.java:181)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.timedAccept(NioSocketImpl.java:710)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.accept(NioSocketImpl.java:752)
        at java.base@16.0.2/java.net.ServerSocket.implAccept(ServerSocket.java:681)
        at java.base@16.0.2/java.net.ServerSocket.platformImplAccept(ServerSocket.java:647)
        at java.base@16.0.2/java.net.ServerSocket.implAccept(ServerSocket.java:623)
        at java.base@16.0.2/java.net.ServerSocket.implAccept(ServerSocket.java:580)
        at java.base@16.0.2/java.net.ServerSocket.accept(ServerSocket.java:538)
        at app//org.apache.solr.client.solrj.cloud.SocketProxy$Acceptor.run(SocketProxy.java:445)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  107) Thread[id=20086, name=Connector-Scheduler-186620a7-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  108) Thread[id=19948, name=qtp2035113691-19948, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  109) Thread[id=20080, name=searcherExecutor-13043-thread-1, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  110) Thread[id=20007, name=qtp1593956388-20007, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  111) Thread[id=20077, name=watches-12979-thread-1-processing-127.0.0.1:51012_kvc%2Fm collection1_shard1_replica_n1 collection1 shard1 core_node2, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkStateReader$CacheCleaner.run(ZkStateReader.java:2332)
        at java.base@16.0.2/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
        at java.base@16.0.2/java.util.concurrent.FutureTask.run(FutureTask.java:264)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$267/0x0000000800f8c7a0.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  112) Thread[id=19949, name=qtp2035113691-19949, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  113) Thread[id=20127, name=watches-13017-thread-4, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  114) Thread[id=20110, name=zkCallback-13015-thread-2, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  115) Thread[id=20057, name=CloserThread, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/java.lang.Object.wait(Native Method)
        at java.base@16.0.2/java.lang.Object.wait(Object.java:320)
        at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2409)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 105 threads leaked from SUITE scope at org.apache.solr.cloud.HttpPartitionOnCommitTest: 
   1) Thread[id=20126, name=watches-13017-thread-3, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   2) Thread[id=19981, name=qtp847956012-19981, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   3) Thread[id=20027, name=zkConnectionManagerCallback-13016-thread-1, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   4) Thread[id=20006, name=qtp1593956388-20006, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   5) Thread[id=20038, name=qtp1024122880-20038, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   6) Thread[id=20031, name=SocketProxy-Acceptor-51041, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.Net.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.park(NioSocketImpl.java:181)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.timedAccept(NioSocketImpl.java:710)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.accept(NioSocketImpl.java:752)
        at java.base@16.0.2/java.net.ServerSocket.implAccept(ServerSocket.java:681)
        at java.base@16.0.2/java.net.ServerSocket.platformImplAccept(ServerSocket.java:647)
        at java.base@16.0.2/java.net.ServerSocket.implAccept(ServerSocket.java:623)
        at java.base@16.0.2/java.net.ServerSocket.implAccept(ServerSocket.java:580)
        at java.base@16.0.2/java.net.ServerSocket.accept(ServerSocket.java:538)
        at app//org.apache.solr.client.solrj.cloud.SocketProxy$Acceptor.run(SocketProxy.java:445)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   7) Thread[id=20155, name=zkCallback-13015-thread-5, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   8) Thread[id=19950, name=qtp2035113691-19950, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   9) Thread[id=19980, name=qtp847956012-19980-acceptor-0@449afce5-ServerConnector@186620a7{HTTP/1.1, (http/1.1, h2c)}{127.0.0.1:51028}, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.Net.accept(Native Method)
        at java.base@16.0.2/sun.nio.ch.ServerSocketChannelImpl.implAccept(ServerSocketChannelImpl.java:424)
        at java.base@16.0.2/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:390)
        at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388)
        at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:702)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  10) Thread[id=20014, name=Session-HouseKeeper-7b5e93a0-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  11) Thread[id=20018, name=httpShardExecutor-13009-thread-1, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$267/0x0000000800f8c7a0.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  12) Thread[id=19955, name=qtp2035113691-19955, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  13) Thread[id=20152, name=zkCallback-12977-thread-4, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  14) Thread[id=20128, name=h2sc-12973-thread-6, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  15) Thread[id=19978, name=qtp847956012-19978, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  16) Thread[id=20000, name=CloserThread, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/java.lang.Object.wait(Native Method)
        at java.base@16.0.2/java.lang.Object.wait(Object.java:320)
        at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2409)
  17) Thread[id=20088, name=watches-12998-thread-2-processing-127.0.0.1:51027_kvc%2Fm collection1_shard1_replica_n3 collection1 shard1 core_node4, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkStateReader$CacheCleaner.run(ZkStateReader.java:2332)
        at java.base@16.0.2/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
        at java.base@16.0.2/java.util.concurrent.FutureTask.run(FutureTask.java:264)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$267/0x0000000800f8c7a0.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  18) Thread[id=20030, name=coreContainerWorkExecutor-13006-thread-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  19) Thread[id=20040, name=qtp1024122880-20040, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  20) Thread[id=20113, name=updateExecutor-13010-thread-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  21) Thread[id=19951, name=qtp2035113691-19951-acceptor-0@7b549fb1-ServerConnector@7d72d52a{HTTP/1.1, (http/1.1, h2c)}{127.0.0.1:51013}, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.Net.accept(Native Method)
        at java.base@16.0.2/sun.nio.ch.ServerSocketChannelImpl.implAccept(ServerSocketChannelImpl.java:424)
        at java.base@16.0.2/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:390)
        at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388)
        at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:702)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  22) Thread[id=19982, name=qtp847956012-19982, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  23) Thread[id=20058, name=coreContainerWorkExecutor-13025-thread-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  24) Thread[id=20009, name=qtp1593956388-20009-acceptor-0@7d6c2e71-ServerConnector@27960ae1{HTTP/1.1, (http/1.1, h2c)}{127.0.0.1:51043}, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.Net.accept(Native Method)
        at java.base@16.0.2/sun.nio.ch.ServerSocketChannelImpl.implAccept(ServerSocketChannelImpl.java:424)
        at java.base@16.0.2/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:390)
        at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388)
        at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:702)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  25) Thread[id=20010, name=qtp1593956388-20010, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  26) Thread[id=19952, name=qtp2035113691-19952, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  27) Thread[id=20001, name=coreContainerWorkExecutor-12987-thread-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  28) Thread[id=19953, name=qtp2035113691-19953, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  29) Thread[id=19983, name=qtp847956012-19983, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  30) Thread[id=20153, name=zkCallback-13015-thread-4, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  31) Thread[id=19999, name=zkCallback-12996-thread-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  32) Thread[id=20039, name=qtp1024122880-20039, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  33) Thread[id=20098, name=updateExecutor-12972-thread-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  34) Thread[id=20116, name=SocketProxy-Response-51013:51097, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.Net.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.park(NioSocketImpl.java:181)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.timedRead(NioSocketImpl.java:285)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.implRead(NioSocketImpl.java:309)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.read(NioSocketImpl.java:350)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl$1.read(NioSocketImpl.java:803)
        at java.base@16.0.2/java.net.Socket$SocketInputStream.read(Socket.java:976)
        at java.base@16.0.2/java.io.InputStream.read(InputStream.java:218)
        at app//org.apache.solr.client.solrj.cloud.SocketProxy$Bridge$Pump.run(SocketProxy.java:375)
  35) Thread[id=19998, name=zkConnectionManagerCallback-12997-thread-1, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  36) Thread[id=19977, name=qtp847956012-19977, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  37) Thread[id=20093, name=recoveryExecutor-12993-thread-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  38) Thread[id=20029, name=CloserThread, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/java.lang.Object.wait(Native Method)
        at java.base@16.0.2/java.lang.Object.wait(Object.java:320)
        at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2409)
  39) Thread[id=19972, name=CloserThread, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/java.lang.Object.wait(Native Method)
        at java.base@16.0.2/java.lang.Object.wait(Object.java:320)
        at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2409)
  40) Thread[id=20158, name=zkCallback-12996-thread-4, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  41) Thread[id=20159, name=zkCallback-13034-thread-3, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  42) Thread[id=20101, name=h2sc-12973-thread-3, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  43) Thread[id=19954, name=qtp2035113691-19954, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  44) Thread[id=20041, name=qtp1024122880-20041, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  45) Thread[id=20160, name=zkCallback-12996-thread-5, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  46) Thread[id=20055, name=zkConnectionManagerCallback-13035-thread-1, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  47) Thread[id=20115, name=SocketProxy-Request-51097:51013, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.Net.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.park(NioSocketImpl.java:181)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.timedRead(NioSocketImpl.java:285)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.implRead(NioSocketImpl.java:309)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.read(NioSocketImpl.java:350)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl$1.read(NioSocketImpl.java:803)
        at java.base@16.0.2/java.net.Socket$SocketInputStream.read(Socket.java:976)
        at java.base@16.0.2/java.io.InputStream.read(InputStream.java:218)
        at app//org.apache.solr.client.solrj.cloud.SocketProxy$Bridge$Pump.run(SocketProxy.java:375)
  48) Thread[id=20104, name=watches-12979-thread-3, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  49) Thread[id=20036, name=qtp1024122880-20036, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  50) Thread[id=20081, name=zkCallback-12977-thread-3, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  51) Thread[id=19979, name=qtp847956012-19979, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  52) Thread[id=20121, name=h2sc-12973-thread-5, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  53) Thread[id=20157, name=zkCallback-13034-thread-2, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  54) Thread[id=20114, name=recoveryExecutor-13012-thread-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  55) Thread[id=20118, name=zkCallback-13015-thread-3, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  56) Thread[id=20125, name=watches-12998-thread-3, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  57) Thread[id=19985, name=Session-HouseKeeper-5d93b66e-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  58) Thread[id=20078, name=zkCallback-12977-thread-2, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  59) Thread[id=20156, name=zkCallback-12977-thread-6, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  60) Thread[id=20011, name=qtp1593956388-20011, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  61) Thread[id=20034, name=qtp1024122880-20034, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  62) Thread[id=20049, name=h2sc-13030-thread-1, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$267/0x0000000800f8c7a0.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  63) Thread[id=20013, name=qtp1593956388-20013, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  64) Thread[id=20095, name=SocketProxy-Response-51013:51082, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.Net.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.park(NioSocketImpl.java:181)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.timedRead(NioSocketImpl.java:285)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.implRead(NioSocketImpl.java:309)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.read(NioSocketImpl.java:350)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl$1.read(NioSocketImpl.java:803)
        at java.base@16.0.2/java.net.Socket$SocketInputStream.read(Socket.java:976)
        at java.base@16.0.2/java.io.InputStream.read(InputStream.java:218)
        at app//org.apache.solr.client.solrj.cloud.SocketProxy$Bridge$Pump.run(SocketProxy.java:375)
  65) Thread[id=20035, name=qtp1024122880-20035, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  66) Thread[id=20037, name=qtp1024122880-20037-acceptor-0@2be775ce-ServerConnector@7a90fc7b{HTTP/1.1, (http/1.1, h2c)}{127.0.0.1:51053}, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.Net.accept(Native Method)
        at java.base@16.0.2/sun.nio.ch.ServerSocketChannelImpl.implAccept(ServerSocketChannelImpl.java:424)
        at java.base@16.0.2/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:390)
        at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388)
        at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:702)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  67) Thread[id=20089, name=zkCallback-12996-thread-2, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  68) Thread[id=20046, name=httpShardExecutor-13028-thread-1, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$267/0x0000000800f8c7a0.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  69) Thread[id=19963, name=h2sc-12973-thread-1, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$267/0x0000000800f8c7a0.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  70) Thread[id=20094, name=SocketProxy-Request-51082:51013, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.Net.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.park(NioSocketImpl.java:181)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.timedRead(NioSocketImpl.java:285)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.implRead(NioSocketImpl.java:309)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.read(NioSocketImpl.java:350)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl$1.read(NioSocketImpl.java:803)
        at java.base@16.0.2/java.net.Socket$SocketInputStream.read(Socket.java:976)
        at java.base@16.0.2/java.io.InputStream.read(InputStream.java:218)
        at app//org.apache.solr.client.solrj.cloud.SocketProxy$Bridge$Pump.run(SocketProxy.java:375)
  71) Thread[id=20120, name=h2sc-12973-thread-4, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  72) Thread[id=20042, name=Session-HouseKeeper-1ed1a01d-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  73) Thread[id=20124, name=watches-13017-thread-2, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  74) Thread[id=20150, name=OverseerStateUpdate-72169010312052745-127.0.0.1:51012_kvc%2Fm-n_0000000001, state=TIMED_WAITING, group=Overseer state updater.]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:488)
        at app//org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:320)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  75) Thread[id=19992, name=h2sc-12992-thread-1, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$267/0x0000000800f8c7a0.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  76) Thread[id=20056, name=zkCallback-13034-thread-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  77) Thread[id=20154, name=zkCallback-12977-thread-5, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  78) Thread[id=20021, name=h2sc-13011-thread-1, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$267/0x0000000800f8c7a0.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  79) Thread[id=20112, name=searcherExecutor-13056-thread-1, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  80) Thread[id=20087, name=watches-12998-thread-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  81) Thread[id=19984, name=qtp847956012-19984, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  82) Thread[id=20075, name=Connector-Scheduler-7d72d52a-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  83) Thread[id=20091, name=searcherExecutor-13049-thread-1, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  84) Thread[id=20107, name=Connector-Scheduler-27960ae1-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  85) Thread[id=19973, name=coreContainerWorkExecutor-12968-thread-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  86) Thread[id=20083, name=watches-12979-thread-2, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  87) Thread[id=19970, name=zkCallback-12977-thread-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  88) Thread[id=20109, name=watches-13017-thread-1-processing-127.0.0.1:51041_kvc%2Fm collection1_shard1_replica_n5 collection1 shard1 core_node6, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkStateReader$CacheCleaner.run(ZkStateReader.java:2332)
        at java.base@16.0.2/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
        at java.base@16.0.2/java.util.concurrent.FutureTask.run(FutureTask.java:264)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$267/0x0000000800f8c7a0.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  89) Thread[id=20099, name=HttpClient@59ad64a7-scheduler-1, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  90) Thread[id=20100, name=h2sc-12973-thread-2, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  91) Thread[id=20012, name=qtp1593956388-20012, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  92) Thread[id=20119, name=updateExecutor-12972-thread-2, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  93) Thread[id=19989, name=httpShardExecutor-12990-thread-1, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$267/0x0000000800f8c7a0.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  94) Thread[id=19974, name=SocketProxy-Acceptor-51012, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.Net.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.park(NioSocketImpl.java:181)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.timedAccept(NioSocketImpl.java:710)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.accept(NioSocketImpl.java:752)
        at java.base@16.0.2/java.net.ServerSocket.implAccept(ServerSocket.java:681)
        at java.base@16.0.2/java.net.ServerSocket.platformImplAccept(ServerSocket.java:647)
        at java.base@16.0.2/java.net.ServerSocket.implAccept(ServerSocket.java:623)
        at java.base@16.0.2/java.net.ServerSocket.implAccept(ServerSocket.java:580)
        at java.base@16.0.2/java.net.ServerSocket.accept(ServerSocket.java:538)
        at app//org.apache.solr.client.solrj.cloud.SocketProxy$Acceptor.run(SocketProxy.java:445)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  95) Thread[id=20097, name=zkCallback-12996-thread-3, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  96) Thread[id=20161, name=zkCallback-13034-thread-4, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  97) Thread[id=20151, name=OverseerCollectionConfigSetProcessor-72169010312052745-127.0.0.1:51012_kvc%2Fm-n_0000000001, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:488)
        at app//org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:266)
        at app//org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:274)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  98) Thread[id=19960, name=httpShardExecutor-12971-thread-1, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$267/0x0000000800f8c7a0.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  99) Thread[id=20002, name=SocketProxy-Acceptor-51027, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.Net.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.park(NioSocketImpl.java:181)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.timedAccept(NioSocketImpl.java:710)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.accept(NioSocketImpl.java:752)
        at java.base@16.0.2/java.net.ServerSocket.implAccept(ServerSocket.java:681)
        at java.base@16.0.2/java.net.ServerSocket.platformImplAccept(ServerSocket.java:647)
        at java.base@16.0.2/java.net.ServerSocket.implAccept(ServerSocket.java:623)
        at java.base@16.0.2/java.net.ServerSocket.implAccept(ServerSocket.java:580)
        at java.base@16.0.2/java.net.ServerSocket.accept(ServerSocket.java:538)
        at app//org.apache.solr.client.solrj.cloud.SocketProxy$Acceptor.run(SocketProxy.java:445)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  100) Thread[id=20008, name=qtp1593956388-20008, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  101) Thread[id=20129, name=h2sc-12973-thread-7, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  102) Thread[id=19969, name=zkConnectionManagerCallback-12978-thread-1, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  103) Thread[id=20092, name=updateExecutor-12991-thread-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  104) Thread[id=19956, name=Session-HouseKeeper-3e0774b4-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  105) Thread[id=20028, name=zkCallback-13015-thread-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  106) Thread[id=20059, name=SocketProxy-Acceptor-51052, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.Net.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.park(NioSocketImpl.java:181)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.timedAccept(NioSocketImpl.java:710)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.accept(NioSocketImpl.java:752)
        at java.base@16.0.2/java.net.ServerSocket.implAccept(ServerSocket.java:681)
        at java.base@16.0.2/java.net.ServerSocket.platformImplAccept(ServerSocket.java:647)
        at java.base@16.0.2/java.net.ServerSocket.implAccept(ServerSocket.java:623)
        at java.base@16.0.2/java.net.ServerSocket.implAccept(ServerSocket.java:580)
        at java.base@16.0.2/java.net.ServerSocket.accept(ServerSocket.java:538)
        at app//org.apache.solr.client.solrj.cloud.SocketProxy$Acceptor.run(SocketProxy.java:445)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  107) Thread[id=20086, name=Connector-Scheduler-186620a7-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  108) Thread[id=19948, name=qtp2035113691-19948, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  109) Thread[id=20080, name=searcherExecutor-13043-thread-1, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  110) Thread[id=20007, name=qtp1593956388-20007, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  111) Thread[id=20077, name=watches-12979-thread-1-processing-127.0.0.1:51012_kvc%2Fm collection1_shard1_replica_n1 collection1 shard1 core_node2, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkStateReader$CacheCleaner.run(ZkStateReader.java:2332)
        at java.base@16.0.2/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
        at java.base@16.0.2/java.util.concurrent.FutureTask.run(FutureTask.java:264)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$267/0x0000000800f8c7a0.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  112) Thread[id=19949, name=qtp2035113691-19949, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  113) Thread[id=20127, name=watches-13017-thread-4, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  114) Thread[id=20110, name=zkCallback-13015-thread-2, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  115) Thread[id=20057, name=CloserThread, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/java.lang.Object.wait(Native Method)
        at java.base@16.0.2/java.lang.Object.wait(Object.java:320)
        at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2409)
	at __randomizedtesting.SeedInfo.seed([EC1DB1CE26ED2B60]:0)


FAILED:  org.apache.solr.cloud.HttpPartitionOnCommitTest.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=20126, name=watches-13017-thread-3, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   2) Thread[id=19981, name=qtp847956012-19981, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   3) Thread[id=20027, name=zkConnectionManagerCallback-13016-thread-1, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   4) Thread[id=20006, name=qtp1593956388-20006, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   5) Thread[id=20038, name=qtp1024122880-20038, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   6) Thread[id=20155, name=zkCallback-13015-thread-5, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   7) Thread[id=19950, name=qtp2035113691-19950, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   8) Thread[id=19980, name=qtp847956012-19980, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   9) Thread[id=20014, name=Session-HouseKeeper-7b5e93a0-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  10) Thread[id=20018, name=httpShardExecutor-13009-thread-1, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$267/0x0000000800f8c7a0.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  11) Thread[id=19955, name=qtp2035113691-19955, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  12) Thread[id=20152, name=zkCallback-12977-thread-4, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  13) Thread[id=20128, name=h2sc-12973-thread-6, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  14) Thread[id=19978, name=qtp847956012-19978, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  15) Thread[id=20000, name=CloserThread, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/java.lang.Object.wait(Native Method)
        at java.base@16.0.2/java.lang.Object.wait(Object.java:320)
        at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2409)
  16) Thread[id=20088, name=watches-12998-thread-2, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  17) Thread[id=20030, name=coreContainerWorkExecutor-13006-thread-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  18) Thread[id=20040, name=qtp1024122880-20040, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  19) Thread[id=20113, name=updateExecutor-13010-thread-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  20) Thread[id=19951, name=qtp2035113691-19951, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  21) Thread[id=19982, name=qtp847956012-19982, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  22) Thread[id=20058, name=coreContainerWorkExecutor-13025-thread-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  23) Thread[id=20009, name=qtp1593956388-20009, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  24) Thread[id=20010, name=qtp1593956388-20010, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  25) Thread[id=19952, name=qtp2035113691-19952, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  26) Thread[id=20001, name=coreContainerWorkExecutor-12987-thread-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  27) Thread[id=19953, name=qtp2035113691-19953, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  28) Thread[id=19983, name=qtp847956012-19983, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  29) Thread[id=20153, name=zkCallback-13015-thread-4, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  30) Thread[id=19999, name=zkCallback-12996-thread-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  31) Thread[id=20039, name=qtp1024122880-20039, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  32) Thread[id=20098, name=updateExecutor-12972-thread-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  33) Thread[id=20116, name=SocketProxy-Response-51013:51097, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.Net.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.park(NioSocketImpl.java:181)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.timedRead(NioSocketImpl.java:285)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.implRead(NioSocketImpl.java:309)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.read(NioSocketImpl.java:350)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl$1.read(NioSocketImpl.java:803)
        at java.base@16.0.2/java.net.Socket$SocketInputStream.read(Socket.java:976)
        at java.base@16.0.2/java.io.InputStream.read(InputStream.java:218)
        at app//org.apache.solr.client.solrj.cloud.SocketProxy$Bridge$Pump.run(SocketProxy.java:375)
  34) Thread[id=19998, name=zkConnectionManagerCallback-12997-thread-1, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  35) Thread[id=19977, name=qtp847956012-19977, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  36) Thread[id=20093, name=recoveryExecutor-12993-thread-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  37) Thread[id=20029, name=CloserThread, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/java.lang.Object.wait(Native Method)
        at java.base@16.0.2/java.lang.Object.wait(Object.java:320)
        at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2409)
  38) Thread[id=19972, name=CloserThread, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/java.lang.Object.wait(Native Method)
        at java.base@16.0.2/java.lang.Object.wait(Object.java:320)
        at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2409)
  39) Thread[id=20158, name=zkCallback-12996-thread-4, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  40) Thread[id=20159, name=zkCallback-13034-thread-3, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  41) Thread[id=20101, name=h2sc-12973-thread-3, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  42) Thread[id=19954, name=qtp2035113691-19954, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  43) Thread[id=20041, name=qtp1024122880-20041, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  44) Thread[id=20160, name=zkCallback-12996-thread-5, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  45) Thread[id=20055, name=zkConnectionManagerCallback-13035-thread-1, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  46) Thread[id=20115, name=SocketProxy-Request-51097:51013, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.Net.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.park(NioSocketImpl.java:181)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.timedRead(NioSocketImpl.java:285)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.implRead(NioSocketImpl.java:309)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.read(NioSocketImpl.java:350)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl$1.read(NioSocketImpl.java:803)
        at java.base@16.0.2/java.net.Socket$SocketInputStream.read(Socket.java:976)
        at java.base@16.0.2/java.io.InputStream.read(InputStream.java:218)
        at app//org.apache.solr.client.solrj.cloud.SocketProxy$Bridge$Pump.run(SocketProxy.java:375)
  47) Thread[id=20104, name=watches-12979-thread-3, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  48) Thread[id=20036, name=qtp1024122880-20036, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  49) Thread[id=20081, name=zkCallback-12977-thread-3, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  50) Thread[id=19979, name=qtp847956012-19979, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  51) Thread[id=20121, name=h2sc-12973-thread-5, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  52) Thread[id=20157, name=zkCallback-13034-thread-2, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  53) Thread[id=20114, name=recoveryExecutor-13012-thread-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  54) Thread[id=20118, name=zkCallback-13015-thread-3, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  55) Thread[id=20125, name=watches-12998-thread-3, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  56) Thread[id=19985, name=Session-HouseKeeper-5d93b66e-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  57) Thread[id=20078, name=zkCallback-12977-thread-2, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  58) Thread[id=20156, name=zkCallback-12977-thread-6, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  59) Thread[id=20011, name=qtp1593956388-20011, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  60) Thread[id=20034, name=qtp1024122880-20034, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  61) Thread[id=20049, name=h2sc-13030-thread-1, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$267/0x0000000800f8c7a0.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  62) Thread[id=20013, name=qtp1593956388-20013, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  63) Thread[id=20095, name=SocketProxy-Response-51013:51082, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.Net.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.park(NioSocketImpl.java:181)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.timedRead(NioSocketImpl.java:285)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.implRead(NioSocketImpl.java:309)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.read(NioSocketImpl.java:350)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl$1.read(NioSocketImpl.java:803)
        at java.base@16.0.2/java.net.Socket$SocketInputStream.read(Socket.java:976)
        at java.base@16.0.2/java.io.InputStream.read(InputStream.java:218)
        at app//org.apache.solr.client.solrj.cloud.SocketProxy$Bridge$Pump.run(SocketProxy.java:375)
  64) Thread[id=20035, name=qtp1024122880-20035, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  65) Thread[id=20037, name=qtp1024122880-20037, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  66) Thread[id=20089, name=zkCallback-12996-thread-2, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  67) Thread[id=20046, name=httpShardExecutor-13028-thread-1, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$267/0x0000000800f8c7a0.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  68) Thread[id=19963, name=h2sc-12973-thread-1, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$267/0x0000000800f8c7a0.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  69) Thread[id=20094, name=SocketProxy-Request-51082:51013, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.Net.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.park(NioSocketImpl.java:181)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.timedRead(NioSocketImpl.java:285)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.implRead(NioSocketImpl.java:309)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.read(NioSocketImpl.java:350)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl$1.read(NioSocketImpl.java:803)
        at java.base@16.0.2/java.net.Socket$SocketInputStream.read(Socket.java:976)
        at java.base@16.0.2/java.io.InputStream.read(InputStream.java:218)
        at app//org.apache.solr.client.solrj.cloud.SocketProxy$Bridge$Pump.run(SocketProxy.java:375)
  70) Thread[id=20120, name=h2sc-12973-thread-4, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  71) Thread[id=20042, name=Session-HouseKeeper-1ed1a01d-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  72) Thread[id=20124, name=watches-13017-thread-2, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  73) Thread[id=19992, name=h2sc-12992-thread-1, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$267/0x0000000800f8c7a0.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  74) Thread[id=20056, name=zkCallback-13034-thread-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  75) Thread[id=20154, name=zkCallback-12977-thread-5, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  76) Thread[id=20021, name=h2sc-13011-thread-1, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$267/0x0000000800f8c7a0.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  77) Thread[id=20112, name=searcherExecutor-13056-thread-1, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  78) Thread[id=20087, name=watches-12998-thread-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  79) Thread[id=19984, name=qtp847956012-19984, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  80) Thread[id=20075, name=Connector-Scheduler-7d72d52a-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  81) Thread[id=20091, name=searcherExecutor-13049-thread-1, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  82) Thread[id=20107, name=Connector-Scheduler-27960ae1-1, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  83) Thread[id=19973, name=coreContainerWorkExecutor-12968-thread-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  84) Thread[id=20083, name=watches-12979-thread-2, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  85) Thread[id=19970, name=zkCallback-12977-thread-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  86) Thread[id=20109, name=watches-13017-thread-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  87) Thread[id=20099, name=HttpClient@59ad64a7-scheduler-1, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  88) Thread[id=20100, name=h2sc-12973-thread-2, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  89) Thread[id=20012, name=qtp1593956388-20012, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  90) Thread[id=20119, name=updateExecutor-12972-thread-2, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  91) Thread[id=19989, name=httpShardExecutor-12990-thread-1, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$267/0x0000000800f8c7a0.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  92) Thread[id=20097, name=zkCallback-12996-thread-3, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  93) Thread[id=20161, name=zkCallback-13034-thread-4, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  94) Thread[id=19960, name=httpShardExecutor-12971-thread-1, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$267/0x0000000800f8c7a0.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  95) Thread[id=20008, name=qtp1593956388-20008, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  96) Thread[id=20129, name=h2sc-12973-thread-7, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  97) Thread[id=19969, name=zkConnectionManagerCallback-12978-thread-1, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  98) Thread[id=20092, name=updateExecutor-12991-thread-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  99) Thread[id=19956, name=Session-HouseKeeper-3e0774b4-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  100) Thread[id=20028, name=zkCallback-13015-thread-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  101) Thread[id=20086, name=Connector-Scheduler-186620a7-1, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  102) Thread[id=19948, name=qtp2035113691-19948, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  103) Thread[id=20080, name=searcherExecutor-13043-thread-1, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  104) Thread[id=20007, name=qtp1593956388-20007, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  105) Thread[id=20077, name=watches-12979-thread-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  106) Thread[id=19949, name=qtp2035113691-19949, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  107) Thread[id=20127, name=watches-13017-thread-4, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  108) Thread[id=20110, name=zkCallback-13015-thread-2, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  109) Thread[id=20057, name=CloserThread, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/java.lang.Object.wait(Native Method)
        at java.base@16.0.2/java.lang.Object.wait(Object.java:320)
        at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2409)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=20126, name=watches-13017-thread-3, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   2) Thread[id=19981, name=qtp847956012-19981, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   3) Thread[id=20027, name=zkConnectionManagerCallback-13016-thread-1, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   4) Thread[id=20006, name=qtp1593956388-20006, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   5) Thread[id=20038, name=qtp1024122880-20038, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   6) Thread[id=20155, name=zkCallback-13015-thread-5, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   7) Thread[id=19950, name=qtp2035113691-19950, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   8) Thread[id=19980, name=qtp847956012-19980, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   9) Thread[id=20014, name=Session-HouseKeeper-7b5e93a0-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  10) Thread[id=20018, name=httpShardExecutor-13009-thread-1, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$267/0x0000000800f8c7a0.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  11) Thread[id=19955, name=qtp2035113691-19955, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  12) Thread[id=20152, name=zkCallback-12977-thread-4, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  13) Thread[id=20128, name=h2sc-12973-thread-6, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  14) Thread[id=19978, name=qtp847956012-19978, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  15) Thread[id=20000, name=CloserThread, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/java.lang.Object.wait(Native Method)
        at java.base@16.0.2/java.lang.Object.wait(Object.java:320)
        at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2409)
  16) Thread[id=20088, name=watches-12998-thread-2, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  17) Thread[id=20030, name=coreContainerWorkExecutor-13006-thread-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  18) Thread[id=20040, name=qtp1024122880-20040, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  19) Thread[id=20113, name=updateExecutor-13010-thread-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  20) Thread[id=19951, name=qtp2035113691-19951, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  21) Thread[id=19982, name=qtp847956012-19982, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  22) Thread[id=20058, name=coreContainerWorkExecutor-13025-thread-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  23) Thread[id=20009, name=qtp1593956388-20009, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  24) Thread[id=20010, name=qtp1593956388-20010, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  25) Thread[id=19952, name=qtp2035113691-19952, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  26) Thread[id=20001, name=coreContainerWorkExecutor-12987-thread-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  27) Thread[id=19953, name=qtp2035113691-19953, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  28) Thread[id=19983, name=qtp847956012-19983, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  29) Thread[id=20153, name=zkCallback-13015-thread-4, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  30) Thread[id=19999, name=zkCallback-12996-thread-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  31) Thread[id=20039, name=qtp1024122880-20039, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  32) Thread[id=20098, name=updateExecutor-12972-thread-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  33) Thread[id=20116, name=SocketProxy-Response-51013:51097, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.Net.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.park(NioSocketImpl.java:181)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.timedRead(NioSocketImpl.java:285)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.implRead(NioSocketImpl.java:309)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.read(NioSocketImpl.java:350)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl$1.read(NioSocketImpl.java:803)
        at java.base@16.0.2/java.net.Socket$SocketInputStream.read(Socket.java:976)
        at java.base@16.0.2/java.io.InputStream.read(InputStream.java:218)
        at app//org.apache.solr.client.solrj.cloud.SocketProxy$Bridge$Pump.run(SocketProxy.java:375)
  34) Thread[id=19998, name=zkConnectionManagerCallback-12997-thread-1, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  35) Thread[id=19977, name=qtp847956012-19977, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  36) Thread[id=20093, name=recoveryExecutor-12993-thread-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  37) Thread[id=20029, name=CloserThread, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/java.lang.Object.wait(Native Method)
        at java.base@16.0.2/java.lang.Object.wait(Object.java:320)
        at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2409)
  38) Thread[id=19972, name=CloserThread, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/java.lang.Object.wait(Native Method)
        at java.base@16.0.2/java.lang.Object.wait(Object.java:320)
        at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2409)
  39) Thread[id=20158, name=zkCallback-12996-thread-4, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  40) Thread[id=20159, name=zkCallback-13034-thread-3, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  41) Thread[id=20101, name=h2sc-12973-thread-3, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  42) Thread[id=19954, name=qtp2035113691-19954, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  43) Thread[id=20041, name=qtp1024122880-20041, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  44) Thread[id=20160, name=zkCallback-12996-thread-5, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  45) Thread[id=20055, name=zkConnectionManagerCallback-13035-thread-1, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  46) Thread[id=20115, name=SocketProxy-Request-51097:51013, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.Net.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.park(NioSocketImpl.java:181)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.timedRead(NioSocketImpl.java:285)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.implRead(NioSocketImpl.java:309)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.read(NioSocketImpl.java:350)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl$1.read(NioSocketImpl.java:803)
        at java.base@16.0.2/java.net.Socket$SocketInputStream.read(Socket.java:976)
        at java.base@16.0.2/java.io.InputStream.read(InputStream.java:218)
        at app//org.apache.solr.client.solrj.cloud.SocketProxy$Bridge$Pump.run(SocketProxy.java:375)
  47) Thread[id=20104, name=watches-12979-thread-3, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  48) Thread[id=20036, name=qtp1024122880-20036, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  49) Thread[id=20081, name=zkCallback-12977-thread-3, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  50) Thread[id=19979, name=qtp847956012-19979, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  51) Thread[id=20121, name=h2sc-12973-thread-5, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  52) Thread[id=20157, name=zkCallback-13034-thread-2, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  53) Thread[id=20114, name=recoveryExecutor-13012-thread-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  54) Thread[id=20118, name=zkCallback-13015-thread-3, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  55) Thread[id=20125, name=watches-12998-thread-3, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  56) Thread[id=19985, name=Session-HouseKeeper-5d93b66e-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  57) Thread[id=20078, name=zkCallback-12977-thread-2, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  58) Thread[id=20156, name=zkCallback-12977-thread-6, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  59) Thread[id=20011, name=qtp1593956388-20011, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  60) Thread[id=20034, name=qtp1024122880-20034, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  61) Thread[id=20049, name=h2sc-13030-thread-1, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$267/0x0000000800f8c7a0.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  62) Thread[id=20013, name=qtp1593956388-20013, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  63) Thread[id=20095, name=SocketProxy-Response-51013:51082, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.Net.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.park(NioSocketImpl.java:181)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.timedRead(NioSocketImpl.java:285)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.implRead(NioSocketImpl.java:309)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.read(NioSocketImpl.java:350)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl$1.read(NioSocketImpl.java:803)
        at java.base@16.0.2/java.net.Socket$SocketInputStream.read(Socket.java:976)
        at java.base@16.0.2/java.io.InputStream.read(InputStream.java:218)
        at app//org.apache.solr.client.solrj.cloud.SocketProxy$Bridge$Pump.run(SocketProxy.java:375)
  64) Thread[id=20035, name=qtp1024122880-20035, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  65) Thread[id=20037, name=qtp1024122880-20037, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  66) Thread[id=20089, name=zkCallback-12996-thread-2, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  67) Thread[id=20046, name=httpShardExecutor-13028-thread-1, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$267/0x0000000800f8c7a0.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  68) Thread[id=19963, name=h2sc-12973-thread-1, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$267/0x0000000800f8c7a0.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  69) Thread[id=20094, name=SocketProxy-Request-51082:51013, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.Net.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.park(NioSocketImpl.java:181)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.timedRead(NioSocketImpl.java:285)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.implRead(NioSocketImpl.java:309)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl.read(NioSocketImpl.java:350)
        at java.base@16.0.2/sun.nio.ch.NioSocketImpl$1.read(NioSocketImpl.java:803)
        at java.base@16.0.2/java.net.Socket$SocketInputStream.read(Socket.java:976)
        at java.base@16.0.2/java.io.InputStream.read(InputStream.java:218)
        at app//org.apache.solr.client.solrj.cloud.SocketProxy$Bridge$Pump.run(SocketProxy.java:375)
  70) Thread[id=20120, name=h2sc-12973-thread-4, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  71) Thread[id=20042, name=Session-HouseKeeper-1ed1a01d-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  72) Thread[id=20124, name=watches-13017-thread-2, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  73) Thread[id=19992, name=h2sc-12992-thread-1, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$267/0x0000000800f8c7a0.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  74) Thread[id=20056, name=zkCallback-13034-thread-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  75) Thread[id=20154, name=zkCallback-12977-thread-5, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  76) Thread[id=20021, name=h2sc-13011-thread-1, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$267/0x0000000800f8c7a0.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  77) Thread[id=20112, name=searcherExecutor-13056-thread-1, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  78) Thread[id=20087, name=watches-12998-thread-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  79) Thread[id=19984, name=qtp847956012-19984, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  80) Thread[id=20075, name=Connector-Scheduler-7d72d52a-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  81) Thread[id=20091, name=searcherExecutor-13049-thread-1, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  82) Thread[id=20107, name=Connector-Scheduler-27960ae1-1, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  83) Thread[id=19973, name=coreContainerWorkExecutor-12968-thread-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  84) Thread[id=20083, name=watches-12979-thread-2, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  85) Thread[id=19970, name=zkCallback-12977-thread-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  86) Thread[id=20109, name=watches-13017-thread-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  87) Thread[id=20099, name=HttpClient@59ad64a7-scheduler-1, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  88) Thread[id=20100, name=h2sc-12973-thread-2, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  89) Thread[id=20012, name=qtp1593956388-20012, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  90) Thread[id=20119, name=updateExecutor-12972-thread-2, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  91) Thread[id=19989, name=httpShardExecutor-12990-thread-1, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$267/0x0000000800f8c7a0.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  92) Thread[id=20097, name=zkCallback-12996-thread-3, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  93) Thread[id=20161, name=zkCallback-13034-thread-4, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  94) Thread[id=19960, name=httpShardExecutor-12971-thread-1, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$267/0x0000000800f8c7a0.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  95) Thread[id=20008, name=qtp1593956388-20008, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  96) Thread[id=20129, name=h2sc-12973-thread-7, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  97) Thread[id=19969, name=zkConnectionManagerCallback-12978-thread-1, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  98) Thread[id=20092, name=updateExecutor-12991-thread-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  99) Thread[id=19956, name=Session-HouseKeeper-3e0774b4-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  100) Thread[id=20028, name=zkCallback-13015-thread-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  101) Thread[id=20086, name=Connector-Scheduler-186620a7-1, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  102) Thread[id=19948, name=qtp2035113691-19948, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  103) Thread[id=20080, name=searcherExecutor-13043-thread-1, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  104) Thread[id=20007, name=qtp1593956388-20007, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  105) Thread[id=20077, name=watches-12979-thread-1, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  106) Thread[id=19949, name=qtp2035113691-19949, state=RUNNABLE, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edc298.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  107) Thread[id=20127, name=watches-13017-thread-4, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  108) Thread[id=20110, name=zkCallback-13015-thread-2, state=TIMED_WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
  109) Thread[id=20057, name=CloserThread, state=WAITING, group=TGRP-HttpPartitionOnCommitTest]
        at java.base@16.0.2/java.lang.Object.wait(Native Method)
        at java.base@16.0.2/java.lang.Object.wait(Object.java:320)
        at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2409)
	at __randomizedtesting.SeedInfo.seed([EC1DB1CE26ED2B60]:0)


FAILED:  org.apache.solr.cloud.ReplaceNodeTest.testGoodSpreadDuringAssignWithNoTarget

Error Message:
java.lang.AssertionError: There should be no more replicas on the sourceNode after a replaceNode request. expected:<[]> but was:<[core_node19:{
  "core":"replacenodetest_coll_shard1_replica_n7",
  "core_node_name":"core_node19",
  "node_name":"127.0.0.1:52169_solr",
  "base_url":"https://127.0.0.1:52169/solr",
  "state":"active",
  "collection":"replacenodetest_coll",
  "shard":"shard1",
  "type":"NRT",
  "force_set_state":"false"}, core_node23:{
  "core":"replacenodetest_coll_shard2_replica_n11",
  "core_node_name":"core_node23",
  "node_name":"127.0.0.1:52169_solr",
  "base_url":"https://127.0.0.1:52169/solr",
  "state":"active",
  "collection":"replacenodetest_coll",
  "shard":"shard2",
  "type":"NRT",
  "force_set_state":"false"}, core_node15:{
  "core":"replacenodetest_coll_shard3_replica_n3",
  "core_node_name":"core_node15",
  "node_name":"127.0.0.1:52169_solr",
  "base_url":"https://127.0.0.1:52169/solr",
  "state":"active",
  "collection":"replacenodetest_coll",
  "shard":"shard3",
  "type":"NRT",
  "force_set_state":"false"}, core_node13:{
  "core":"replacenodetest_coll_shard4_replica_n1",
  "core_node_name":"core_node13",
  "node_name":"127.0.0.1:52169_solr",
  "base_url":"https://127.0.0.1:52169/solr",
  "state":"active",
  "collection":"replacenodetest_coll",
  "shard":"shard4",
  "type":"NRT",
  "force_set_state":"false"}]>

Stack Trace:
java.lang.AssertionError: There should be no more replicas on the sourceNode after a replaceNode request. expected:<[]> but was:<[core_node19:{
  "core":"replacenodetest_coll_shard1_replica_n7",
  "core_node_name":"core_node19",
  "node_name":"127.0.0.1:52169_solr",
  "base_url":"https://127.0.0.1:52169/solr",
  "state":"active",
  "collection":"replacenodetest_coll",
  "shard":"shard1",
  "type":"NRT",
  "force_set_state":"false"}, core_node23:{
  "core":"replacenodetest_coll_shard2_replica_n11",
  "core_node_name":"core_node23",
  "node_name":"127.0.0.1:52169_solr",
  "base_url":"https://127.0.0.1:52169/solr",
  "state":"active",
  "collection":"replacenodetest_coll",
  "shard":"shard2",
  "type":"NRT",
  "force_set_state":"false"}, core_node15:{
  "core":"replacenodetest_coll_shard3_replica_n3",
  "core_node_name":"core_node15",
  "node_name":"127.0.0.1:52169_solr",
  "base_url":"https://127.0.0.1:52169/solr",
  "state":"active",
  "collection":"replacenodetest_coll",
  "shard":"shard3",
  "type":"NRT",
  "force_set_state":"false"}, core_node13:{
  "core":"replacenodetest_coll_shard4_replica_n1",
  "core_node_name":"core_node13",
  "node_name":"127.0.0.1:52169_solr",
  "base_url":"https://127.0.0.1:52169/solr",
  "state":"active",
  "collection":"replacenodetest_coll",
  "shard":"shard4",
  "type":"NRT",
  "force_set_state":"false"}]>
	at __randomizedtesting.SeedInfo.seed([EC1DB1CE26ED2B60:C27665F4138EFA14]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.junit.Assert.failNotEquals(Assert.java:835)
	at org.junit.Assert.assertEquals(Assert.java:120)
	at org.apache.solr.cloud.ReplaceNodeTest.testGoodSpreadDuringAssignWithNoTarget(ReplaceNodeTest.java:272)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:78)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:567)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base/java.lang.Thread.run(Thread.java:831)


FAILED:  org.apache.solr.schema.TestBulkSchemaConcurrent.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.schema.TestBulkSchemaConcurrent: 
   1) Thread[id=9053, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestBulkSchemaConcurrent]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:328)
        at app//org.apache.solr.cloud.ZkController.pathExists(ZkController.java:1228)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:73)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getLines(AbstractAnalysisFactory.java:302)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getWordSet(AbstractAnalysisFactory.java:293)
        at app//org.apache.lucene.analysis.miscellaneous.WordDelimiterGraphFilterFactory.inform(WordDelimiterGraphFilterFactory.java:120)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:762)
        at app//org.apache.solr.core.SolrResourceLoader.inform(SolrResourceLoader.java:749)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:189)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:119)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:279)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:342)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:253)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$610/0x0000000801143898.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:49)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:249)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1850)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3394)
        at app//org.apache.solr.core.SolrCore$$Lambda$735/0x000000080129de58.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2761)
        at app//org.apache.solr.cloud.ZkController$$Lambda$871/0x00000008012f7d10.run(Unknown Source)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.schema.TestBulkSchemaConcurrent: 
   1) Thread[id=9053, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestBulkSchemaConcurrent]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:328)
        at app//org.apache.solr.cloud.ZkController.pathExists(ZkController.java:1228)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:73)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getLines(AbstractAnalysisFactory.java:302)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getWordSet(AbstractAnalysisFactory.java:293)
        at app//org.apache.lucene.analysis.miscellaneous.WordDelimiterGraphFilterFactory.inform(WordDelimiterGraphFilterFactory.java:120)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:762)
        at app//org.apache.solr.core.SolrResourceLoader.inform(SolrResourceLoader.java:749)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:189)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:119)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:279)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:342)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:253)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$610/0x0000000801143898.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:49)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:249)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1850)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3394)
        at app//org.apache.solr.core.SolrCore$$Lambda$735/0x000000080129de58.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2761)
        at app//org.apache.solr.cloud.ZkController$$Lambda$871/0x00000008012f7d10.run(Unknown Source)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
	at __randomizedtesting.SeedInfo.seed([EC1DB1CE26ED2B60]:0)


FAILED:  org.apache.solr.security.hadoop.SaslZkACLProviderTest.testSaslZkACLProvider

Error Message:
org.apache.solr.common.SolrException: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper localhost:57338 within 30000 ms

Stack Trace:
org.apache.solr.common.SolrException: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper localhost:57338 within 30000 ms
	at __randomizedtesting.SeedInfo.seed([EC1DB1CE26ED2B60:E8E9E716BFA9886E]:0)
	at app//org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:213)
	at app//org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:126)
	at app//org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:109)
	at app//org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:101)
	at app//org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:97)
	at app//org.apache.solr.security.hadoop.SaslZkACLProviderTest$SolrZkClientWithACLs.<init>(SaslZkACLProviderTest.java:181)
	at app//org.apache.solr.security.hadoop.SaslZkACLProviderTest.setUp(SaslZkACLProviderTest.java:93)
	at java.base@16.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base@16.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:78)
	at java.base@16.0.2/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base@16.0.2/java.lang.reflect.Method.invoke(Method.java:567)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:980)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
Caused by: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper localhost:57338 within 30000 ms
	at org.apache.solr.common.cloud.ConnectionManager.waitForConnected(ConnectionManager.java:297)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:204)
	... 51 more


FAILED:  org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=1115, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:361)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:75)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getLines(AbstractAnalysisFactory.java:302)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getWordSet(AbstractAnalysisFactory.java:293)
        at app//org.apache.lucene.analysis.en.AbstractWordsFileFilterFactory.inform(AbstractWordsFileFilterFactory.java:88)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:762)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1470)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1328)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1307)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:654)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:188)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:119)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:279)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:342)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:253)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$629/0x000000080114ec50.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:49)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:249)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1850)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3394)
        at app//org.apache.solr.core.SolrCore$$Lambda$739/0x0000000801214000.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2761)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1217/0x00000008016337d8.run(Unknown Source)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=1115, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:361)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:75)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getLines(AbstractAnalysisFactory.java:302)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getWordSet(AbstractAnalysisFactory.java:293)
        at app//org.apache.lucene.analysis.en.AbstractWordsFileFilterFactory.inform(AbstractWordsFileFilterFactory.java:88)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:762)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1470)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1328)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1307)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:654)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:188)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:119)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:279)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:342)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:253)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$629/0x000000080114ec50.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:49)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:249)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1850)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3394)
        at app//org.apache.solr.core.SolrCore$$Lambda$739/0x0000000801214000.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2761)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1217/0x00000008016337d8.run(Unknown Source)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
	at __randomizedtesting.SeedInfo.seed([EC1DB1CE26ED2B60]:0)


FAILED:  org.apache.solr.security.AuthWithShardHandlerFactoryOverrideTest.collectionTest

Error Message:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:60017/solr: no servers hosting shard: shard4

Stack Trace:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:60017/solr: no servers hosting shard: shard4
	at __randomizedtesting.SeedInfo.seed([EC1DB1CE26ED2B60:394CB7ECB5A8609D]:0)
	at app//org.apache.solr.client.solrj.impl.Http2SolrClient.processErrorsAndResponse(Http2SolrClient.java:822)
	at app//org.apache.solr.client.solrj.impl.Http2SolrClient.processErrorsAndResponse(Http2SolrClient.java:485)
	at app//org.apache.solr.client.solrj.impl.Http2SolrClient.request(Http2SolrClient.java:453)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:223)
	at app//org.apache.solr.security.AuthWithShardHandlerFactoryOverrideTest.collectionTest(AuthWithShardHandlerFactoryOverrideTest.java:92)
	at java.base@16.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base@16.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:78)
	at java.base@16.0.2/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base@16.0.2/java.lang.reflect.Method.invoke(Method.java:567)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)

[JENKINS] Solr-main-MacOSX (64bit/jdk-17.0.2) - Build # 888 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/888/
Java: 64bit/jdk-17.0.2 -XX:+UseCompressedOops -XX:+UseSerialGC

7 tests failed.
FAILED:  org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=1066, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@17.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:361)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:75)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getSnowballWordSet(AbstractAnalysisFactory.java:321)
        at app//org.apache.lucene.analysis.en.AbstractWordsFileFilterFactory.inform(AbstractWordsFileFilterFactory.java:90)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:762)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1470)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1319)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1307)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:654)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:188)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:119)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:279)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:342)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:253)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$635/0x000000080114ea18.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:49)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:249)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1850)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3394)
        at app//org.apache.solr.core.SolrCore$$Lambda$745/0x0000000801214000.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2761)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1204/0x0000000801675e08.run(Unknown Source)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=1066, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@17.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:361)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:75)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getSnowballWordSet(AbstractAnalysisFactory.java:321)
        at app//org.apache.lucene.analysis.en.AbstractWordsFileFilterFactory.inform(AbstractWordsFileFilterFactory.java:90)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:762)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1470)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1319)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1307)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:654)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:188)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:119)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:279)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:342)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:253)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$635/0x000000080114ea18.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:49)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:249)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1850)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3394)
        at app//org.apache.solr.core.SolrCore$$Lambda$745/0x0000000801214000.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2761)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1204/0x0000000801675e08.run(Unknown Source)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
	at __randomizedtesting.SeedInfo.seed([E9083B63C876CDD1]:0)


FAILED:  org.apache.solr.schema.TestManagedSchemaAPI.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=3995, name=qtp374583833-3995, state=RUNNABLE, group=TGRP-TestManagedSchemaAPI], registration stack trace below.

Stack Trace:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=3995, name=qtp374583833-3995, state=RUNNABLE, group=TGRP-TestManagedSchemaAPI], registration stack trace below.
	at java.base@17.0.2/java.lang.Thread.getStackTrace(Thread.java:1610)
	at app//com.carrotsearch.randomizedtesting.RandomizedContext.closeAtEnd(RandomizedContext.java:157)
	at app//org.apache.lucene.util.LuceneTestCase.closeAfterSuite(LuceneTestCase.java:798)
	at app//org.apache.lucene.util.LuceneTestCase.wrapDirectory(LuceneTestCase.java:1504)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1368)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1350)
	at app//org.apache.solr.core.MockDirectoryFactory.create(MockDirectoryFactory.java:51)
	at app//org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:368)
	at app//org.apache.solr.core.SolrCore.initSnapshotMetaDataManager(SolrCore.java:606)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1119)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1048)
	at app//org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1562)
	at app//org.apache.solr.core.CoreContainer.create(CoreContainer.java:1428)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:389)
	at app//org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:422)
	at app//org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:205)
	at app//org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:207)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:941)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:893)
	at app//org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:584)
	at app//org.apache.solr.servlet.SolrDispatchFilter.dispatch(SolrDispatchFilter.java:239)
	at app//org.apache.solr.servlet.SolrDispatchFilter.lambda$doFilter$0(SolrDispatchFilter.java:207)
	at app//org.apache.solr.servlet.ServletUtils.traceHttpRequestExecution2(ServletUtils.java:257)
	at app//org.apache.solr.servlet.ServletUtils.rateLimitRequest(ServletUtils.java:227)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:202)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:184)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:187)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1434)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
	at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)
	at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1349)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
	at app//org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:713)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.server.Server.handle(Server.java:516)
	at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:400)
	at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:645)
	at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:392)
	at app//org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:352)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
	at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
Caused by: java.lang.AssertionError: Directory not closed: MockDirectoryWrapper(ByteBuffersDirectory@11084f4e lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@264bec0c)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.lucene.util.CloseableDirectory.close(CloseableDirectory.java:43)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:789)
	at com.carrotsearch.randomizedtesting.RandomizedContext.closeResources(RandomizedContext.java:225)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.afterAlways(RandomizedRunner.java:805)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	... 1 more


FAILED:  org.apache.solr.schema.TestManagedSchemaAPI.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=3995, name=qtp374583833-3995, state=RUNNABLE, group=TGRP-TestManagedSchemaAPI], registration stack trace below.

Stack Trace:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=3995, name=qtp374583833-3995, state=RUNNABLE, group=TGRP-TestManagedSchemaAPI], registration stack trace below.
	at java.base@17.0.2/java.lang.Thread.getStackTrace(Thread.java:1610)
	at app//com.carrotsearch.randomizedtesting.RandomizedContext.closeAtEnd(RandomizedContext.java:157)
	at app//org.apache.lucene.util.LuceneTestCase.closeAfterSuite(LuceneTestCase.java:798)
	at app//org.apache.lucene.util.LuceneTestCase.wrapDirectory(LuceneTestCase.java:1504)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1368)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1350)
	at app//org.apache.solr.core.MockDirectoryFactory.create(MockDirectoryFactory.java:51)
	at app//org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:368)
	at app//org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:412)
	at app//org.apache.solr.core.SolrCore.initIndex(SolrCore.java:856)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1123)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1048)
	at app//org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1562)
	at app//org.apache.solr.core.CoreContainer.create(CoreContainer.java:1428)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:389)
	at app//org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:422)
	at app//org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:205)
	at app//org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:207)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:941)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:893)
	at app//org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:584)
	at app//org.apache.solr.servlet.SolrDispatchFilter.dispatch(SolrDispatchFilter.java:239)
	at app//org.apache.solr.servlet.SolrDispatchFilter.lambda$doFilter$0(SolrDispatchFilter.java:207)
	at app//org.apache.solr.servlet.ServletUtils.traceHttpRequestExecution2(ServletUtils.java:257)
	at app//org.apache.solr.servlet.ServletUtils.rateLimitRequest(ServletUtils.java:227)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:202)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:184)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:187)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1434)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
	at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)
	at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1349)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
	at app//org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:713)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.server.Server.handle(Server.java:516)
	at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:400)
	at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:645)
	at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:392)
	at app//org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:352)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
	at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
Caused by: java.lang.AssertionError: Directory not closed: MockDirectoryWrapper(ByteBuffersDirectory@60db5ade lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6d25f56)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.lucene.util.CloseableDirectory.close(CloseableDirectory.java:43)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:789)
	at com.carrotsearch.randomizedtesting.RandomizedContext.closeResources(RandomizedContext.java:225)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.afterAlways(RandomizedRunner.java:805)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	... 1 more


FAILED:  org.apache.solr.schema.TestManagedSchemaAPI.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=3995, name=qtp374583833-3995, state=RUNNABLE, group=TGRP-TestManagedSchemaAPI], registration stack trace below.

Stack Trace:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=3995, name=qtp374583833-3995, state=RUNNABLE, group=TGRP-TestManagedSchemaAPI], registration stack trace below.
	at java.base@17.0.2/java.lang.Thread.getStackTrace(Thread.java:1610)
	at app//com.carrotsearch.randomizedtesting.RandomizedContext.closeAtEnd(RandomizedContext.java:157)
	at app//org.apache.lucene.util.LuceneTestCase.closeAfterSuite(LuceneTestCase.java:798)
	at app//org.apache.lucene.util.LuceneTestCase.wrapDirectory(LuceneTestCase.java:1504)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1368)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1350)
	at app//org.apache.solr.core.MockDirectoryFactory.create(MockDirectoryFactory.java:51)
	at app//org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:368)
	at app//org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:108)
	at app//org.apache.solr.core.SolrCore.initIndex(SolrCore.java:895)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1123)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1048)
	at app//org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1562)
	at app//org.apache.solr.core.CoreContainer.create(CoreContainer.java:1428)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:389)
	at app//org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:422)
	at app//org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:205)
	at app//org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:207)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:941)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:893)
	at app//org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:584)
	at app//org.apache.solr.servlet.SolrDispatchFilter.dispatch(SolrDispatchFilter.java:239)
	at app//org.apache.solr.servlet.SolrDispatchFilter.lambda$doFilter$0(SolrDispatchFilter.java:207)
	at app//org.apache.solr.servlet.ServletUtils.traceHttpRequestExecution2(ServletUtils.java:257)
	at app//org.apache.solr.servlet.ServletUtils.rateLimitRequest(ServletUtils.java:227)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:202)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:184)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:187)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1434)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
	at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)
	at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1349)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
	at app//org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:713)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.server.Server.handle(Server.java:516)
	at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:400)
	at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:645)
	at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:392)
	at app//org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:352)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
	at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
Caused by: java.lang.AssertionError: Directory not closed: MockDirectoryWrapper(ByteBuffersDirectory@42f0fd9c lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@25705a39)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.lucene.util.CloseableDirectory.close(CloseableDirectory.java:43)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:789)
	at com.carrotsearch.randomizedtesting.RandomizedContext.closeResources(RandomizedContext.java:225)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.afterAlways(RandomizedRunner.java:805)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	... 1 more


FAILED:  org.apache.solr.schema.TestManagedSchemaAPI.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.schema.TestManagedSchemaAPI: 
   1) Thread[id=4101, name=searcherExecutor-3663-thread-1-processing-127.0.0.1:59827_solr, state=WAITING, group=TGRP-TestManagedSchemaAPI]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
        at java.base@17.0.2/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:230)
        at app//org.apache.solr.core.SolrCore.lambda$new$2(SolrCore.java:1142)
        at app//org.apache.solr.core.SolrCore$$Lambda$722/0x00000008011f7db0.call(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.FutureTask.run(FutureTask.java:264)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$264/0x0000000800f8f870.run(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   2) Thread[id=4093, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestManagedSchemaAPI]
        at java.base@17.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:361)
        at app//org.apache.solr.schema.ZkIndexSchemaReader.updateSchema(ZkIndexSchemaReader.java:176)
        at app//org.apache.solr.schema.ZkIndexSchemaReader.refreshSchemaFromZk(ZkIndexSchemaReader.java:165)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.inform(ManagedIndexSchemaFactory.java:541)
        at app//org.apache.solr.core.SolrResourceLoader.inform(SolrResourceLoader.java:728)
        at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1155)
        at app//org.apache.solr.core.SolrCore.reload(SolrCore.java:780)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1857)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3394)
        at app//org.apache.solr.core.SolrCore$$Lambda$745/0x0000000801214000.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2761)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1204/0x0000000801675e08.run(Unknown Source)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.schema.TestManagedSchemaAPI: 
   1) Thread[id=4101, name=searcherExecutor-3663-thread-1-processing-127.0.0.1:59827_solr, state=WAITING, group=TGRP-TestManagedSchemaAPI]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
        at java.base@17.0.2/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:230)
        at app//org.apache.solr.core.SolrCore.lambda$new$2(SolrCore.java:1142)
        at app//org.apache.solr.core.SolrCore$$Lambda$722/0x00000008011f7db0.call(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.FutureTask.run(FutureTask.java:264)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$264/0x0000000800f8f870.run(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   2) Thread[id=4093, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestManagedSchemaAPI]
        at java.base@17.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:361)
        at app//org.apache.solr.schema.ZkIndexSchemaReader.updateSchema(ZkIndexSchemaReader.java:176)
        at app//org.apache.solr.schema.ZkIndexSchemaReader.refreshSchemaFromZk(ZkIndexSchemaReader.java:165)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.inform(ManagedIndexSchemaFactory.java:541)
        at app//org.apache.solr.core.SolrResourceLoader.inform(SolrResourceLoader.java:728)
        at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1155)
        at app//org.apache.solr.core.SolrCore.reload(SolrCore.java:780)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1857)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3394)
        at app//org.apache.solr.core.SolrCore$$Lambda$745/0x0000000801214000.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2761)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1204/0x0000000801675e08.run(Unknown Source)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
	at __randomizedtesting.SeedInfo.seed([E9083B63C876CDD1]:0)


FAILED:  org.apache.solr.security.AuthWithShardHandlerFactoryOverrideTest.collectionTest

Error Message:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:63406/solr: no servers hosting shard: shard4

Stack Trace:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:63406/solr: no servers hosting shard: shard4
	at __randomizedtesting.SeedInfo.seed([E9083B63C876CDD1:3C593D415B33862C]:0)
	at app//org.apache.solr.client.solrj.impl.Http2SolrClient.processErrorsAndResponse(Http2SolrClient.java:822)
	at app//org.apache.solr.client.solrj.impl.Http2SolrClient.processErrorsAndResponse(Http2SolrClient.java:485)
	at app//org.apache.solr.client.solrj.impl.Http2SolrClient.request(Http2SolrClient.java:453)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:223)
	at app//org.apache.solr.security.AuthWithShardHandlerFactoryOverrideTest.collectionTest(AuthWithShardHandlerFactoryOverrideTest.java:92)
	at java.base@17.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base@17.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base@17.0.2/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base@17.0.2/java.lang.reflect.Method.invoke(Method.java:568)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)


FAILED:  org.apache.solr.cloud.TestRebalanceLeaders.testRebalanceLeaders

Error Message:
org.apache.solr.client.solrj.SolrServerException: Timeout occurred while waiting response from server at: https://127.0.0.1:62036/solr

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occurred while waiting response from server at: https://127.0.0.1:62036/solr
	at __randomizedtesting.SeedInfo.seed([E9083B63C876CDD1:F7E70993F0DD81A1]:0)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:724)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:405)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:371)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1248)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:969)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:896)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:223)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:238)
	at app//org.apache.solr.cloud.TestRebalanceLeaders.rebalanceLeaderUsingSolrJAPI(TestRebalanceLeaders.java:431)
	at app//org.apache.solr.cloud.TestRebalanceLeaders.sendRebalanceCommand(TestRebalanceLeaders.java:328)
	at app//org.apache.solr.cloud.TestRebalanceLeaders.testRebalanceLeaders(TestRebalanceLeaders.java:145)
	at java.base@17.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base@17.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base@17.0.2/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base@17.0.2/java.lang.reflect.Method.invoke(Method.java:568)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.base/sun.nio.ch.NioSocketImpl.timedRead(NioSocketImpl.java:283)
	at java.base/sun.nio.ch.NioSocketImpl.implRead(NioSocketImpl.java:309)
	at java.base/sun.nio.ch.NioSocketImpl.read(NioSocketImpl.java:350)
	at java.base/sun.nio.ch.NioSocketImpl$1.read(NioSocketImpl.java:803)
	at java.base/java.net.Socket$SocketInputStream.read(Socket.java:966)
	at java.base/sun.security.ssl.SSLSocketInputRecord.read(SSLSocketInputRecord.java:478)
	at java.base/sun.security.ssl.SSLSocketInputRecord.readHeader(SSLSocketInputRecord.java:472)
	at java.base/sun.security.ssl.SSLSocketInputRecord.bytesInCompletePacket(SSLSocketInputRecord.java:70)
	at java.base/sun.security.ssl.SSLSocketImpl.readApplicationRecord(SSLSocketImpl.java:1455)
	at java.base/sun.security.ssl.SSLSocketImpl$AppInputStream.read(SSLSocketImpl.java:1059)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:280)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:157)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:576)
	... 58 more

[JENKINS] Solr-main-MacOSX (64bit/jdk-11.0.13) - Build # 887 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/887/
Java: 64bit/jdk-11.0.13 -XX:+UseCompressedOops -XX:+UseSerialGC

2 tests failed.
FAILED:  org.apache.solr.cloud.HttpPartitionTest.test

Error Message:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:54398: Underlying core creation failed while creating collection: collDoRecoveryOnRestart

Stack Trace:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:54398: Underlying core creation failed while creating collection: collDoRecoveryOnRestart
	at __randomizedtesting.SeedInfo.seed([E8DC079AE4DE67E2:608838404A220A1A]:0)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:715)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at app//org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1193)
	at app//org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:2099)
	at app//org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:2137)
	at app//org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:2025)
	at app//org.apache.solr.cloud.HttpPartitionTest.testDoRecoveryOnRestart(HttpPartitionTest.java:163)
	at app//org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:133)
	at java.base@11.0.13/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base@11.0.13/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base@11.0.13/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base@11.0.13/java.lang.reflect.Method.invoke(Method.java:566)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1164)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1135)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)


FAILED:  org.apache.solr.cloud.TestRebalanceLeaders.testRebalanceLeaders

Error Message:
org.apache.solr.client.solrj.SolrServerException: Timeout occurred while waiting response from server at: http://127.0.0.1:50234/solr

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occurred while waiting response from server at: http://127.0.0.1:50234/solr
	at __randomizedtesting.SeedInfo.seed([E8DC079AE4DE67E2:F633356ADC752B92]:0)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:724)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:405)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:371)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1248)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:969)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:896)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:223)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:238)
	at app//org.apache.solr.cloud.TestRebalanceLeaders.rebalanceLeaderUsingSolrJAPI(TestRebalanceLeaders.java:431)
	at app//org.apache.solr.cloud.TestRebalanceLeaders.sendRebalanceCommand(TestRebalanceLeaders.java:328)
	at app//org.apache.solr.cloud.TestRebalanceLeaders.testRebalanceLeaders(TestRebalanceLeaders.java:145)
	at java.base@11.0.13/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base@11.0.13/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base@11.0.13/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base@11.0.13/java.lang.reflect.Method.invoke(Method.java:566)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.base/java.net.SocketInputStream.socketRead0(Native Method)
	at java.base/java.net.SocketInputStream.socketRead(SocketInputStream.java:115)
	at java.base/java.net.SocketInputStream.read(SocketInputStream.java:168)
	at java.base/java.net.SocketInputStream.read(SocketInputStream.java:140)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:280)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:157)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:576)
	... 58 more

[JENKINS] Solr-main-MacOSX (64bit/jdk-16.0.2) - Build # 886 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/886/
Java: 64bit/jdk-16.0.2 -XX:-UseCompressedOops -XX:+UseParallelGC

10 tests failed.
FAILED:  org.apache.solr.cloud.HttpPartitionWithTlogReplicasTest.test

Error Message:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:56601: Underlying core creation failed while creating collection: c8n_1x2_leader_session_loss

Stack Trace:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:56601: Underlying core creation failed while creating collection: c8n_1x2_leader_session_loss
	at __randomizedtesting.SeedInfo.seed([976EE8908B6C26BD:1F3AD74A25904B45]:0)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:715)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at app//org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1193)
	at app//org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:2099)
	at app//org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:2137)
	at app//org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:2025)
	at app//org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollectionRetry(AbstractFullDistribZkTestBase.java:2404)
	at app//org.apache.solr.cloud.HttpPartitionTest.testLeaderZkSessionLoss(HttpPartitionTest.java:421)
	at app//org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:149)
	at java.base@16.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base@16.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:78)
	at java.base@16.0.2/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base@16.0.2/java.lang.reflect.Method.invoke(Method.java:567)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1164)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1135)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)


FAILED:  org.apache.solr.cloud.TestConfigSetsAPI.testUploadLegacyManagedSchemaFile

Error Message:
java.util.concurrent.TimeoutException: Timed out waiting for all collections to be fully removed.

Stack Trace:
java.util.concurrent.TimeoutException: Timed out waiting for all collections to be fully removed.
	at org.apache.solr.cloud.MiniSolrCloudCluster.deleteAllCollections(MiniSolrCloudCluster.java:661)
	at org.apache.solr.cloud.TestConfigSetsAPI.tearDown(TestConfigSetsAPI.java:133)
	at jdk.internal.reflect.GeneratedMethodAccessor63.invoke(Unknown Source)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:567)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:1004)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base/java.lang.Thread.run(Thread.java:831)


FAILED:  org.apache.solr.cloud.TestConfigSetsAPI.testCreateWithTrust

Error Message:
java.util.concurrent.TimeoutException: Timed out waiting for all collections to be fully removed.

Stack Trace:
java.util.concurrent.TimeoutException: Timed out waiting for all collections to be fully removed.
	at __randomizedtesting.SeedInfo.seed([976EE8908B6C26BD:7E2E1DEAC2EB441E]:0)
	at org.apache.solr.cloud.MiniSolrCloudCluster.deleteAllCollections(MiniSolrCloudCluster.java:661)
	at org.apache.solr.cloud.TestConfigSetsAPI.tearDown(TestConfigSetsAPI.java:133)
	at jdk.internal.reflect.GeneratedMethodAccessor63.invoke(Unknown Source)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:567)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:1004)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base/java.lang.Thread.run(Thread.java:831)


FAILED:  org.apache.solr.cloud.TestConfigSetsAPI.testSingleFileNewConfigV2

Error Message:
java.util.concurrent.TimeoutException: Timed out waiting for all collections to be fully removed.

Stack Trace:
java.util.concurrent.TimeoutException: Timed out waiting for all collections to be fully removed.
	at __randomizedtesting.SeedInfo.seed([976EE8908B6C26BD:CA81679C03974035]:0)
	at org.apache.solr.cloud.MiniSolrCloudCluster.deleteAllCollections(MiniSolrCloudCluster.java:661)
	at org.apache.solr.cloud.TestConfigSetsAPI.tearDown(TestConfigSetsAPI.java:133)
	at jdk.internal.reflect.GeneratedMethodAccessor63.invoke(Unknown Source)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:567)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:1004)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base/java.lang.Thread.run(Thread.java:831)


FAILED:  org.apache.solr.cloud.TestConfigSetsAPI.testSingleFileOverwriteV2

Error Message:
java.util.concurrent.TimeoutException: Timed out waiting for all collections to be fully removed.

Stack Trace:
java.util.concurrent.TimeoutException: Timed out waiting for all collections to be fully removed.
	at __randomizedtesting.SeedInfo.seed([976EE8908B6C26BD:2EFDFE5626EDC0FB]:0)
	at org.apache.solr.cloud.MiniSolrCloudCluster.deleteAllCollections(MiniSolrCloudCluster.java:661)
	at org.apache.solr.cloud.TestConfigSetsAPI.tearDown(TestConfigSetsAPI.java:133)
	at jdk.internal.reflect.GeneratedMethodAccessor63.invoke(Unknown Source)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:567)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:1004)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base/java.lang.Thread.run(Thread.java:831)


FAILED:  org.apache.solr.cloud.TestConfigSetsAPI.testSingleFileOverwriteV1

Error Message:
java.util.concurrent.TimeoutException: Timed out waiting for all collections to be fully removed.

Stack Trace:
java.util.concurrent.TimeoutException: Timed out waiting for all collections to be fully removed.
	at __randomizedtesting.SeedInfo.seed([976EE8908B6C26BD:3849F638DCD49233]:0)
	at org.apache.solr.cloud.MiniSolrCloudCluster.deleteAllCollections(MiniSolrCloudCluster.java:661)
	at org.apache.solr.cloud.TestConfigSetsAPI.tearDown(TestConfigSetsAPI.java:133)
	at jdk.internal.reflect.GeneratedMethodAccessor63.invoke(Unknown Source)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:567)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:1004)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base/java.lang.Thread.run(Thread.java:831)


FAILED:  org.apache.solr.cloud.TestConfigSetsAPI.testOverwriteV1

Error Message:
java.util.concurrent.TimeoutException: Timed out waiting for all collections to be fully removed.

Stack Trace:
java.util.concurrent.TimeoutException: Timed out waiting for all collections to be fully removed.
	at __randomizedtesting.SeedInfo.seed([976EE8908B6C26BD:49CF63727DFE0997]:0)
	at org.apache.solr.cloud.MiniSolrCloudCluster.deleteAllCollections(MiniSolrCloudCluster.java:661)
	at org.apache.solr.cloud.TestConfigSetsAPI.tearDown(TestConfigSetsAPI.java:133)
	at jdk.internal.reflect.GeneratedMethodAccessor63.invoke(Unknown Source)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:567)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:1004)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base/java.lang.Thread.run(Thread.java:831)


FAILED:  org.apache.solr.cloud.TestConfigSetsAPI.testSingleFileNewConfigV1

Error Message:
java.util.concurrent.TimeoutException: Timed out waiting for all collections to be fully removed.

Stack Trace:
java.util.concurrent.TimeoutException: Timed out waiting for all collections to be fully removed.
	at __randomizedtesting.SeedInfo.seed([976EE8908B6C26BD:8B7555EDE0C48E5C]:0)
	at org.apache.solr.cloud.MiniSolrCloudCluster.deleteAllCollections(MiniSolrCloudCluster.java:661)
	at org.apache.solr.cloud.TestConfigSetsAPI.tearDown(TestConfigSetsAPI.java:133)
	at jdk.internal.reflect.GeneratedMethodAccessor63.invoke(Unknown Source)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:567)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:1004)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base/java.lang.Thread.run(Thread.java:831)


FAILED:  org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=1096, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:328)
        at app//org.apache.solr.cloud.ZkController.pathExists(ZkController.java:1228)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:73)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getLines(AbstractAnalysisFactory.java:302)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getWordSet(AbstractAnalysisFactory.java:293)
        at app//org.apache.lucene.analysis.en.AbstractWordsFileFilterFactory.inform(AbstractWordsFileFilterFactory.java:88)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:762)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1470)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1319)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1307)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:654)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:188)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:119)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:279)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:342)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:253)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$654/0x00000008011d30d0.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:49)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:249)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1850)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3394)
        at app//org.apache.solr.core.SolrCore$$Lambda$764/0x00000008012957d8.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2761)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1217/0x0000000801688450.run(Unknown Source)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=1096, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:328)
        at app//org.apache.solr.cloud.ZkController.pathExists(ZkController.java:1228)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:73)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getLines(AbstractAnalysisFactory.java:302)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getWordSet(AbstractAnalysisFactory.java:293)
        at app//org.apache.lucene.analysis.en.AbstractWordsFileFilterFactory.inform(AbstractWordsFileFilterFactory.java:88)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:762)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1470)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1319)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1307)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:654)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:188)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:119)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:279)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:342)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:253)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$654/0x00000008011d30d0.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:49)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:249)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1850)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3394)
        at app//org.apache.solr.core.SolrCore$$Lambda$764/0x00000008012957d8.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2761)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1217/0x0000000801688450.run(Unknown Source)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
	at __randomizedtesting.SeedInfo.seed([976EE8908B6C26BD]:0)


FAILED:  org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest.test

Error Message:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:52759: Underlying core creation failed while creating collection: testcollection

Stack Trace:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:52759: Underlying core creation failed while creating collection: testcollection
	at __randomizedtesting.SeedInfo.seed([976EE8908B6C26BD:1F3AD74A25904B45]:0)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:715)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:405)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:371)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1248)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:969)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:896)
	at app//org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1193)
	at app//org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:2102)
	at app//org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:2162)
	at app//org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest.test(ChaosMonkeyNothingIsSafeWithPullReplicasTest.java:342)
	at java.base@16.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base@16.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:78)
	at java.base@16.0.2/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base@16.0.2/java.lang.reflect.Method.invoke(Method.java:567)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1164)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1135)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)

[JENKINS] Solr-main-MacOSX (64bit/jdk-11.0.13) - Build # 885 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/885/
Java: 64bit/jdk-11.0.13 -XX:-UseCompressedOops -XX:+UseParallelGC

3 tests failed.
FAILED:  org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest.test

Error Message:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:55478/bv/vq: Underlying core creation failed while creating collection: control_collection

Stack Trace:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:55478/bv/vq: Underlying core creation failed while creating collection: control_collection
	at __randomizedtesting.SeedInfo.seed([A24B1697D7E13264:2A1F294D791D5F9C]:0)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:715)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:405)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:371)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1248)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:969)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:896)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:223)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:238)
	at app//org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:357)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1162)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1135)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)


FAILED:  org.apache.solr.core.snapshots.TestSolrCloudSnapshots.testSnapshots

Error Message:
java.lang.AssertionError: expected:<COMPLETED> but was:<RUNNING>

Stack Trace:
java.lang.AssertionError: expected:<COMPLETED> but was:<RUNNING>
	at __randomizedtesting.SeedInfo.seed([A24B1697D7E13264:94079208A2B8302B]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.junit.Assert.failNotEquals(Assert.java:835)
	at org.junit.Assert.assertEquals(Assert.java:120)
	at org.junit.Assert.assertEquals(Assert.java:146)
	at org.apache.solr.core.snapshots.TestSolrCloudSnapshots.testSnapshots(TestSolrCloudSnapshots.java:218)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base/java.lang.Thread.run(Thread.java:829)


FAILED:  org.apache.solr.schema.TestCloudManagedSchema.test

Error Message:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:60149: Underlying core creation failed while creating collection: control_collection

Stack Trace:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:60149: Underlying core creation failed while creating collection: control_collection
	at __randomizedtesting.SeedInfo.seed([A24B1697D7E13264:2A1F294D791D5F9C]:0)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:715)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:405)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:371)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1248)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:969)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:896)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:223)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:238)
	at app//org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:357)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1162)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1135)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)

[JENKINS] Solr-main-MacOSX (64bit/jdk-16.0.2) - Build # 884 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/884/
Java: 64bit/jdk-16.0.2 -XX:+UseCompressedOops -XX:+UseParallelGC

1 tests failed.
FAILED:  org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=995, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:361)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:75)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getLines(AbstractAnalysisFactory.java:302)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getWordSet(AbstractAnalysisFactory.java:293)
        at app//org.apache.lucene.analysis.en.AbstractWordsFileFilterFactory.inform(AbstractWordsFileFilterFactory.java:88)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:762)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1470)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1319)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1307)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:654)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:188)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:119)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:279)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:342)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:253)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$633/0x000000080114ea18.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:49)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:249)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1850)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3394)
        at app//org.apache.solr.core.SolrCore$$Lambda$743/0x0000000801214000.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2761)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1192/0x00000008015f96a8.run(Unknown Source)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=995, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:361)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:75)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getLines(AbstractAnalysisFactory.java:302)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getWordSet(AbstractAnalysisFactory.java:293)
        at app//org.apache.lucene.analysis.en.AbstractWordsFileFilterFactory.inform(AbstractWordsFileFilterFactory.java:88)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:762)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1470)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1319)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1307)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:654)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:188)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:119)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:279)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:342)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:253)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$633/0x000000080114ea18.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:49)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:249)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1850)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3394)
        at app//org.apache.solr.core.SolrCore$$Lambda$743/0x0000000801214000.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2761)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1192/0x00000008015f96a8.run(Unknown Source)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
	at __randomizedtesting.SeedInfo.seed([2B35231345DA4CD]:0)

[JENKINS] Solr-main-MacOSX (64bit/jdk-11.0.13) - Build # 883 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/883/
Java: 64bit/jdk-11.0.13 -XX:+UseCompressedOops -XX:+UseConcMarkSweepGC

5 tests failed.
FAILED:  org.apache.solr.schema.SchemaApiFailureTest.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=10607, name=qtp2008093538-10607, state=RUNNABLE, group=TGRP-SchemaApiFailureTest], registration stack trace below.

Stack Trace:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=10607, name=qtp2008093538-10607, state=RUNNABLE, group=TGRP-SchemaApiFailureTest], registration stack trace below.
	at java.base@11.0.13/java.lang.Thread.getStackTrace(Thread.java:1602)
	at app//com.carrotsearch.randomizedtesting.RandomizedContext.closeAtEnd(RandomizedContext.java:157)
	at app//org.apache.lucene.util.LuceneTestCase.closeAfterSuite(LuceneTestCase.java:798)
	at app//org.apache.lucene.util.LuceneTestCase.wrapDirectory(LuceneTestCase.java:1504)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1368)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1350)
	at app//org.apache.solr.core.MockDirectoryFactory.create(MockDirectoryFactory.java:51)
	at app//org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:368)
	at app//org.apache.solr.core.SolrCore.initSnapshotMetaDataManager(SolrCore.java:606)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1119)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1048)
	at app//org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1562)
	at app//org.apache.solr.core.CoreContainer.create(CoreContainer.java:1428)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:389)
	at app//org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:422)
	at app//org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:205)
	at app//org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:207)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:941)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:893)
	at app//org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:584)
	at app//org.apache.solr.servlet.SolrDispatchFilter.dispatch(SolrDispatchFilter.java:239)
	at app//org.apache.solr.servlet.SolrDispatchFilter.lambda$doFilter$0(SolrDispatchFilter.java:207)
	at app//org.apache.solr.servlet.ServletUtils.traceHttpRequestExecution2(ServletUtils.java:257)
	at app//org.apache.solr.servlet.ServletUtils.rateLimitRequest(ServletUtils.java:227)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:202)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:184)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:187)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1434)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
	at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)
	at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1349)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
	at app//org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:713)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.server.Server.handle(Server.java:516)
	at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:400)
	at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:645)
	at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:392)
	at app//org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:352)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
	at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
Caused by: java.lang.AssertionError: Directory not closed: MockDirectoryWrapper(ByteBuffersDirectory@624dd900 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@ce8c5b4)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.lucene.util.CloseableDirectory.close(CloseableDirectory.java:43)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:789)
	at com.carrotsearch.randomizedtesting.RandomizedContext.closeResources(RandomizedContext.java:225)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.afterAlways(RandomizedRunner.java:805)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	... 1 more


FAILED:  org.apache.solr.schema.SchemaApiFailureTest.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=10607, name=qtp2008093538-10607, state=RUNNABLE, group=TGRP-SchemaApiFailureTest], registration stack trace below.

Stack Trace:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=10607, name=qtp2008093538-10607, state=RUNNABLE, group=TGRP-SchemaApiFailureTest], registration stack trace below.
	at java.base@11.0.13/java.lang.Thread.getStackTrace(Thread.java:1602)
	at app//com.carrotsearch.randomizedtesting.RandomizedContext.closeAtEnd(RandomizedContext.java:157)
	at app//org.apache.lucene.util.LuceneTestCase.closeAfterSuite(LuceneTestCase.java:798)
	at app//org.apache.lucene.util.LuceneTestCase.wrapDirectory(LuceneTestCase.java:1504)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1368)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1350)
	at app//org.apache.solr.core.MockDirectoryFactory.create(MockDirectoryFactory.java:51)
	at app//org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:368)
	at app//org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:412)
	at app//org.apache.solr.core.SolrCore.initIndex(SolrCore.java:856)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1123)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1048)
	at app//org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1562)
	at app//org.apache.solr.core.CoreContainer.create(CoreContainer.java:1428)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:389)
	at app//org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:422)
	at app//org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:205)
	at app//org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:207)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:941)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:893)
	at app//org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:584)
	at app//org.apache.solr.servlet.SolrDispatchFilter.dispatch(SolrDispatchFilter.java:239)
	at app//org.apache.solr.servlet.SolrDispatchFilter.lambda$doFilter$0(SolrDispatchFilter.java:207)
	at app//org.apache.solr.servlet.ServletUtils.traceHttpRequestExecution2(ServletUtils.java:257)
	at app//org.apache.solr.servlet.ServletUtils.rateLimitRequest(ServletUtils.java:227)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:202)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:184)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:187)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1434)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
	at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)
	at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1349)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
	at app//org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:713)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.server.Server.handle(Server.java:516)
	at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:400)
	at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:645)
	at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:392)
	at app//org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:352)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
	at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
Caused by: java.lang.AssertionError: Directory not closed: MockDirectoryWrapper(ByteBuffersDirectory@5cf945f0 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6867e4b6)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.lucene.util.CloseableDirectory.close(CloseableDirectory.java:43)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:789)
	at com.carrotsearch.randomizedtesting.RandomizedContext.closeResources(RandomizedContext.java:225)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.afterAlways(RandomizedRunner.java:805)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	... 1 more


FAILED:  org.apache.solr.schema.SchemaApiFailureTest.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=10607, name=qtp2008093538-10607, state=RUNNABLE, group=TGRP-SchemaApiFailureTest], registration stack trace below.

Stack Trace:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=10607, name=qtp2008093538-10607, state=RUNNABLE, group=TGRP-SchemaApiFailureTest], registration stack trace below.
	at java.base@11.0.13/java.lang.Thread.getStackTrace(Thread.java:1602)
	at app//com.carrotsearch.randomizedtesting.RandomizedContext.closeAtEnd(RandomizedContext.java:157)
	at app//org.apache.lucene.util.LuceneTestCase.closeAfterSuite(LuceneTestCase.java:798)
	at app//org.apache.lucene.util.LuceneTestCase.wrapDirectory(LuceneTestCase.java:1504)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1368)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1350)
	at app//org.apache.solr.core.MockDirectoryFactory.create(MockDirectoryFactory.java:51)
	at app//org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:368)
	at app//org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:108)
	at app//org.apache.solr.core.SolrCore.initIndex(SolrCore.java:895)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1123)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1048)
	at app//org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1562)
	at app//org.apache.solr.core.CoreContainer.create(CoreContainer.java:1428)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:389)
	at app//org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:422)
	at app//org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:205)
	at app//org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:207)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:941)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:893)
	at app//org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:584)
	at app//org.apache.solr.servlet.SolrDispatchFilter.dispatch(SolrDispatchFilter.java:239)
	at app//org.apache.solr.servlet.SolrDispatchFilter.lambda$doFilter$0(SolrDispatchFilter.java:207)
	at app//org.apache.solr.servlet.ServletUtils.traceHttpRequestExecution2(ServletUtils.java:257)
	at app//org.apache.solr.servlet.ServletUtils.rateLimitRequest(ServletUtils.java:227)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:202)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:184)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:187)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1434)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
	at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)
	at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1349)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
	at app//org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:713)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.server.Server.handle(Server.java:516)
	at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:400)
	at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:645)
	at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:392)
	at app//org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:352)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
	at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
Caused by: java.lang.AssertionError: Directory not closed: MockDirectoryWrapper(NRTCachingDirectory(ByteBuffersDirectory@13ae786d lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6c003c39; maxCacheMB=0.537109375 maxMergeSizeMB=0.52734375))
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.lucene.util.CloseableDirectory.close(CloseableDirectory.java:43)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:789)
	at com.carrotsearch.randomizedtesting.RandomizedContext.closeResources(RandomizedContext.java:225)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.afterAlways(RandomizedRunner.java:805)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	... 1 more


FAILED:  org.apache.solr.schema.SchemaApiFailureTest.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.schema.SchemaApiFailureTest: 
   1) Thread[id=10656, name=searcherExecutor-7957-thread-1-processing-127.0.0.1:62214_solr, state=WAITING, group=TGRP-SchemaApiFailureTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:885)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1039)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1345)
        at java.base@11.0.13/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:232)
        at app//org.apache.solr.core.SolrCore.lambda$new$2(SolrCore.java:1142)
        at app//org.apache.solr.core.SolrCore$$Lambda$456/0x00000008005cf440.call(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.FutureTask.run(FutureTask.java:264)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$257/0x000000080049f440.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   2) Thread[id=10643, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-SchemaApiFailureTest]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:318)
        at app//org.apache.solr.schema.ZkIndexSchemaReader.createSchemaWatcher(ZkIndexSchemaReader.java:104)
        at app//org.apache.solr.schema.ZkIndexSchemaReader.<init>(ZkIndexSchemaReader.java:85)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.inform(ManagedIndexSchemaFactory.java:537)
        at app//org.apache.solr.core.SolrResourceLoader.inform(SolrResourceLoader.java:728)
        at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1155)
        at app//org.apache.solr.core.SolrCore.reload(SolrCore.java:780)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1857)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3394)
        at app//org.apache.solr.core.SolrCore$$Lambda$811/0x0000000800881440.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2761)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1265/0x0000000800d08840.run(Unknown Source)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.schema.SchemaApiFailureTest: 
   1) Thread[id=10656, name=searcherExecutor-7957-thread-1-processing-127.0.0.1:62214_solr, state=WAITING, group=TGRP-SchemaApiFailureTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:885)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1039)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1345)
        at java.base@11.0.13/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:232)
        at app//org.apache.solr.core.SolrCore.lambda$new$2(SolrCore.java:1142)
        at app//org.apache.solr.core.SolrCore$$Lambda$456/0x00000008005cf440.call(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.FutureTask.run(FutureTask.java:264)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$257/0x000000080049f440.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   2) Thread[id=10643, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-SchemaApiFailureTest]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:318)
        at app//org.apache.solr.schema.ZkIndexSchemaReader.createSchemaWatcher(ZkIndexSchemaReader.java:104)
        at app//org.apache.solr.schema.ZkIndexSchemaReader.<init>(ZkIndexSchemaReader.java:85)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.inform(ManagedIndexSchemaFactory.java:537)
        at app//org.apache.solr.core.SolrResourceLoader.inform(SolrResourceLoader.java:728)
        at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1155)
        at app//org.apache.solr.core.SolrCore.reload(SolrCore.java:780)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1857)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3394)
        at app//org.apache.solr.core.SolrCore$$Lambda$811/0x0000000800881440.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2761)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1265/0x0000000800d08840.run(Unknown Source)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
	at __randomizedtesting.SeedInfo.seed([2402FECD13EE1C28]:0)


FAILED:  org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=1192, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:328)
        at app//org.apache.solr.cloud.ZkController.pathExists(ZkController.java:1228)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:73)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getLines(AbstractAnalysisFactory.java:302)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getWordSet(AbstractAnalysisFactory.java:293)
        at app//org.apache.lucene.analysis.en.AbstractWordsFileFilterFactory.inform(AbstractWordsFileFilterFactory.java:88)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:762)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1470)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1319)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1307)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:654)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:188)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:119)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:279)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:342)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:253)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$633/0x0000000800738c40.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:49)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:249)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1850)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3394)
        at app//org.apache.solr.core.SolrCore$$Lambda$743/0x0000000800806040.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2761)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1193/0x0000000800cb8040.run(Unknown Source)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=1192, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:328)
        at app//org.apache.solr.cloud.ZkController.pathExists(ZkController.java:1228)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:73)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getLines(AbstractAnalysisFactory.java:302)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getWordSet(AbstractAnalysisFactory.java:293)
        at app//org.apache.lucene.analysis.en.AbstractWordsFileFilterFactory.inform(AbstractWordsFileFilterFactory.java:88)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:762)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1470)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1319)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1307)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:654)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:188)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:119)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:279)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:342)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:253)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$633/0x0000000800738c40.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:49)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:249)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1850)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3394)
        at app//org.apache.solr.core.SolrCore$$Lambda$743/0x0000000800806040.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2761)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1193/0x0000000800cb8040.run(Unknown Source)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
	at __randomizedtesting.SeedInfo.seed([2402FECD13EE1C28]:0)

[JENKINS] Solr-main-MacOSX (64bit/jdk-18) - Build # 882 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/882/
Java: 64bit/jdk-18 -XX:-UseCompressedOops -XX:+UseSerialGC

5 tests failed.
FAILED:  org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: Timeout occurred while waiting response from server at: http://127.0.0.1:61684/mu/m

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occurred while waiting response from server at: http://127.0.0.1:61684/mu/m
	at __randomizedtesting.SeedInfo.seed([F04D41EF6507FD0C:78197E35CBFB90F4]:0)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:724)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:405)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:371)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1248)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:969)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:896)
	at app//org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1193)
	at app//org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:2102)
	at app//org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:2162)
	at app//org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest.test(ChaosMonkeyNothingIsSafeWithPullReplicasTest.java:342)
	at java.base@18/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
	at java.base@18/java.lang.reflect.Method.invoke(Method.java:577)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1164)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1135)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base@18/java.lang.Thread.run(Thread.java:833)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.base/sun.nio.ch.NioSocketImpl.timedRead(NioSocketImpl.java:280)
	at java.base/sun.nio.ch.NioSocketImpl.implRead(NioSocketImpl.java:306)
	at java.base/sun.nio.ch.NioSocketImpl.read(NioSocketImpl.java:347)
	at java.base/sun.nio.ch.NioSocketImpl$1.read(NioSocketImpl.java:800)
	at java.base/java.net.Socket$SocketInputStream.read(Socket.java:966)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:280)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:157)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:576)
	... 57 more


FAILED:  org.apache.solr.cloud.ReplaceNodeTest.testGoodSpreadDuringAssignWithNoTarget

Error Message:
java.lang.AssertionError: There should be no more replicas on the sourceNode after a replaceNode request. expected:<[]> but was:<[core_node16:{
  "core":"replacenodetest_coll_shard1_replica_n4",
  "core_node_name":"core_node16",
  "node_name":"127.0.0.1:55112_solr",
  "base_url":"http://127.0.0.1:55112/solr",
  "state":"active",
  "collection":"replacenodetest_coll",
  "shard":"shard1",
  "type":"NRT",
  "force_set_state":"false"}, core_node17:{
  "core":"replacenodetest_coll_shard2_replica_n5",
  "core_node_name":"core_node17",
  "node_name":"127.0.0.1:55112_solr",
  "base_url":"http://127.0.0.1:55112/solr",
  "state":"active",
  "collection":"replacenodetest_coll",
  "shard":"shard2",
  "type":"NRT",
  "force_set_state":"false"}, core_node20:{
  "core":"replacenodetest_coll_shard3_replica_n8",
  "leader":"true",
  "core_node_name":"core_node20",
  "node_name":"127.0.0.1:55112_solr",
  "base_url":"http://127.0.0.1:55112/solr",
  "state":"active",
  "collection":"replacenodetest_coll",
  "shard":"shard3",
  "type":"NRT",
  "force_set_state":"false"}, core_node22:{
  "core":"replacenodetest_coll_shard4_replica_n10",
  "leader":"true",
  "core_node_name":"core_node22",
  "node_name":"127.0.0.1:55112_solr",
  "base_url":"http://127.0.0.1:55112/solr",
  "state":"active",
  "collection":"replacenodetest_coll",
  "shard":"shard4",
  "type":"NRT",
  "force_set_state":"false"}]>

Stack Trace:
java.lang.AssertionError: There should be no more replicas on the sourceNode after a replaceNode request. expected:<[]> but was:<[core_node16:{
  "core":"replacenodetest_coll_shard1_replica_n4",
  "core_node_name":"core_node16",
  "node_name":"127.0.0.1:55112_solr",
  "base_url":"http://127.0.0.1:55112/solr",
  "state":"active",
  "collection":"replacenodetest_coll",
  "shard":"shard1",
  "type":"NRT",
  "force_set_state":"false"}, core_node17:{
  "core":"replacenodetest_coll_shard2_replica_n5",
  "core_node_name":"core_node17",
  "node_name":"127.0.0.1:55112_solr",
  "base_url":"http://127.0.0.1:55112/solr",
  "state":"active",
  "collection":"replacenodetest_coll",
  "shard":"shard2",
  "type":"NRT",
  "force_set_state":"false"}, core_node20:{
  "core":"replacenodetest_coll_shard3_replica_n8",
  "leader":"true",
  "core_node_name":"core_node20",
  "node_name":"127.0.0.1:55112_solr",
  "base_url":"http://127.0.0.1:55112/solr",
  "state":"active",
  "collection":"replacenodetest_coll",
  "shard":"shard3",
  "type":"NRT",
  "force_set_state":"false"}, core_node22:{
  "core":"replacenodetest_coll_shard4_replica_n10",
  "leader":"true",
  "core_node_name":"core_node22",
  "node_name":"127.0.0.1:55112_solr",
  "base_url":"http://127.0.0.1:55112/solr",
  "state":"active",
  "collection":"replacenodetest_coll",
  "shard":"shard4",
  "type":"NRT",
  "force_set_state":"false"}]>
	at __randomizedtesting.SeedInfo.seed([F04D41EF6507FD0C:DE2695D550642C78]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.junit.Assert.failNotEquals(Assert.java:835)
	at org.junit.Assert.assertEquals(Assert.java:120)
	at org.apache.solr.cloud.ReplaceNodeTest.testGoodSpreadDuringAssignWithNoTarget(ReplaceNodeTest.java:272)
	at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
	at java.base/java.lang.reflect.Method.invoke(Method.java:577)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base/java.lang.Thread.run(Thread.java:833)


FAILED:  org.apache.solr.cloud.api.collections.TestCollectionAPI.test

Error Message:
org.junit.ComparisonFailure: collection health expected:<[GREEN]> but was:<[RED]>

Stack Trace:
org.junit.ComparisonFailure: collection health expected:<[GREEN]> but was:<[RED]>
	at __randomizedtesting.SeedInfo.seed([F04D41EF6507FD0C:78197E35CBFB90F4]:0)
	at org.junit.Assert.assertEquals(Assert.java:117)
	at org.apache.solr.cloud.api.collections.TestCollectionAPI.clusterStatusWithCollectionHealthState(TestCollectionAPI.java:417)
	at org.apache.solr.cloud.api.collections.TestCollectionAPI.test(TestCollectionAPI.java:95)
	at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
	at java.base/java.lang.reflect.Method.invoke(Method.java:577)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1164)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1135)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base/java.lang.Thread.run(Thread.java:833)


FAILED:  org.apache.solr.metrics.reporters.SolrJmxReporterTest.testClosedCore

Error Message:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=7732, name=TestMBeanThread, state=RUNNABLE, group=TGRP-SolrJmxReporterTest]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=7732, name=TestMBeanThread, state=RUNNABLE, group=TGRP-SolrJmxReporterTest]
	at __randomizedtesting.SeedInfo.seed([F04D41EF6507FD0C:873097CF459E9FFC]:0)
Caused by: java.lang.AssertionError
	at __randomizedtesting.SeedInfo.seed([F04D41EF6507FD0C]:0)
	at org.junit.Assert.fail(Assert.java:87)
	at org.junit.Assert.assertTrue(Assert.java:42)
	at org.junit.Assert.assertNotNull(Assert.java:713)
	at org.junit.Assert.assertNotNull(Assert.java:723)
	at org.apache.solr.metrics.reporters.SolrJmxReporterTest.lambda$testClosedCore$3(SolrJmxReporterTest.java:241)
	at java.base/java.lang.Thread.run(Thread.java:833)


FAILED:  org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=1133, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@18/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:328)
        at app//org.apache.solr.cloud.ZkController.pathExists(ZkController.java:1228)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:73)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getLines(AbstractAnalysisFactory.java:302)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getWordSet(AbstractAnalysisFactory.java:293)
        at app//org.apache.lucene.analysis.util.ElisionFilterFactory.inform(ElisionFilterFactory.java:73)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:762)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1470)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1319)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1307)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:654)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:188)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:119)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:279)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:342)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:253)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$633/0x0000000801137758.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:49)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:249)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1850)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3394)
        at app//org.apache.solr.core.SolrCore$$Lambda$743/0x00000008011f2560.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2761)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1162/0x0000000801503108.run(Unknown Source)
        at java.base@18/java.lang.Thread.run(Thread.java:833)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=1133, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@18/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:328)
        at app//org.apache.solr.cloud.ZkController.pathExists(ZkController.java:1228)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:73)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getLines(AbstractAnalysisFactory.java:302)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getWordSet(AbstractAnalysisFactory.java:293)
        at app//org.apache.lucene.analysis.util.ElisionFilterFactory.inform(ElisionFilterFactory.java:73)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:762)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1470)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1319)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1307)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:654)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:188)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:119)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:279)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:342)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:253)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$633/0x0000000801137758.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:49)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:249)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1850)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3394)
        at app//org.apache.solr.core.SolrCore$$Lambda$743/0x00000008011f2560.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2761)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1162/0x0000000801503108.run(Unknown Source)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
	at __randomizedtesting.SeedInfo.seed([F04D41EF6507FD0C]:0)

[JENKINS] Solr-main-MacOSX (64bit/jdk-11.0.13) - Build # 881 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/881/
Java: 64bit/jdk-11.0.13 -XX:-UseCompressedOops -XX:+UseParallelGC

3 tests failed.
FAILED:  org.apache.solr.cloud.HttpPartitionOnCommitTest.test

Error Message:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:56720/ab_/z: Underlying core creation failed while creating collection: control_collection

Stack Trace:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:56720/ab_/z: Underlying core creation failed while creating collection: control_collection
	at __randomizedtesting.SeedInfo.seed([4DF20FF8399C2FF9:C5A6302297604201]:0)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:715)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:405)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:371)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1248)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:969)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:896)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:223)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:238)
	at app//org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:357)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1162)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1135)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)


FAILED:  org.apache.solr.cloud.TestCloudRecovery2.test

Error Message:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:61943/solr: No registered leader was found after waiting for 4000ms , collection: collection1 slice: shard1 saw state=DocCollection(collection1//collections/collection1/state.json/9 )={
  "pullReplicas":"0",
  "configName":"config",
  "replicationFactor":"2",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node3":{
          "core":"collection1_shard1_replica_n1",
          "core_node_name":"core_node3",
          "node_name":"127.0.0.1:61943_solr",
          "base_url":"http://127.0.0.1:61943/solr",
          "state":"active",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node4":{
          "core":"collection1_shard1_replica_n2",
          "core_node_name":"core_node4",
          "node_name":"127.0.0.1:61944_solr",
          "base_url":"http://127.0.0.1:61944/solr",
          "state":"down",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"2",
  "tlogReplicas":"0"} with live_nodes=[127.0.0.1:61943_solr]

Stack Trace:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:61943/solr: No registered leader was found after waiting for 4000ms , collection: collection1 slice: shard1 saw state=DocCollection(collection1//collections/collection1/state.json/9 )={
  "pullReplicas":"0",
  "configName":"config",
  "replicationFactor":"2",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node3":{
          "core":"collection1_shard1_replica_n1",
          "core_node_name":"core_node3",
          "node_name":"127.0.0.1:61943_solr",
          "base_url":"http://127.0.0.1:61943/solr",
          "state":"active",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node4":{
          "core":"collection1_shard1_replica_n2",
          "core_node_name":"core_node4",
          "node_name":"127.0.0.1:61944_solr",
          "base_url":"http://127.0.0.1:61944/solr",
          "state":"down",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"2",
  "tlogReplicas":"0"} with live_nodes=[127.0.0.1:61943_solr]
	at __randomizedtesting.SeedInfo.seed([4DF20FF8399C2FF9:C5A6302297604201]:0)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:715)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:223)
	at app//org.apache.solr.client.solrj.request.UpdateRequest.commit(UpdateRequest.java:226)
	at app//org.apache.solr.cloud.TestCloudRecovery2.test(TestCloudRecovery2.java:65)
	at java.base@11.0.13/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base@11.0.13/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base@11.0.13/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base@11.0.13/java.lang.reflect.Method.invoke(Method.java:566)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)


FAILED:  org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=1101, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:328)
        at app//org.apache.solr.cloud.ZkController.pathExists(ZkController.java:1228)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:73)
        at app//org.apache.lucene.analysis.synonym.SynonymGraphFilterFactory.loadSynonyms(SynonymGraphFilterFactory.java:191)
        at app//org.apache.lucene.analysis.synonym.SynonymGraphFilterFactory.inform(SynonymGraphFilterFactory.java:161)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:762)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1470)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1328)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1307)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:654)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:188)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:119)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:279)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:342)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:253)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$617/0x000000014236cd08.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:49)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:249)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1850)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3394)
        at app//org.apache.solr.core.SolrCore$$Lambda$727/0x00000001428755b8.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2761)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1189/0x00000001454dbcb0.run(Unknown Source)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=1101, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:328)
        at app//org.apache.solr.cloud.ZkController.pathExists(ZkController.java:1228)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:73)
        at app//org.apache.lucene.analysis.synonym.SynonymGraphFilterFactory.loadSynonyms(SynonymGraphFilterFactory.java:191)
        at app//org.apache.lucene.analysis.synonym.SynonymGraphFilterFactory.inform(SynonymGraphFilterFactory.java:161)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:762)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1470)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1328)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1307)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:654)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:188)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:119)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:279)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:342)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:253)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$617/0x000000014236cd08.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:49)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:249)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1850)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3394)
        at app//org.apache.solr.core.SolrCore$$Lambda$727/0x00000001428755b8.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2761)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1189/0x00000001454dbcb0.run(Unknown Source)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
	at __randomizedtesting.SeedInfo.seed([4DF20FF8399C2FF9]:0)

[JENKINS] Solr-main-MacOSX (64bit/jdk-18) - Build # 880 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/880/
Java: 64bit/jdk-18 -XX:-UseCompressedOops -XX:+UseParallelGC

1 tests failed.
FAILED:  org.apache.solr.cloud.TestStressLiveNodes.testStress

Error Message:
java.lang.AssertionError: iter74 expected:<5> but was:<7>

Stack Trace:
java.lang.AssertionError: iter74 expected:<5> but was:<7>
	at __randomizedtesting.SeedInfo.seed([DA3403E11D239D65:CF2F7E209ED4651E]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.junit.Assert.failNotEquals(Assert.java:835)
	at org.junit.Assert.assertEquals(Assert.java:647)
	at org.apache.solr.cloud.TestStressLiveNodes.testStress(TestStressLiveNodes.java:211)
	at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
	at java.base/java.lang.reflect.Method.invoke(Method.java:577)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base/java.lang.Thread.run(Thread.java:833)

[JENKINS] Solr-main-MacOSX (64bit/jdk-17.0.2) - Build # 879 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/879/
Java: 64bit/jdk-17.0.2 -XX:+UseCompressedOops -XX:+UseParallelGC

2 tests failed.
FAILED:  org.apache.solr.handler.TestSolrConfigHandlerCloud.test

Error Message:
java.lang.AssertionError: {
  "responseHeader":{
    "status":500,
    "QTime":30082},
  "errorMessages":["4 out of 5 the property overlay to be of version 22 within 30 seconds! Failed cores: [http://127.0.0.1:63990/collection1_shard2_replica_n1/, http://127.0.0.1:64061/collection1_shard2_replica_n6/, http://127.0.0.1:64027/collection1_shard1_replica_n2/, http://127.0.0.1:64084/collection1_shard1_replica_n5/]\n"],
  "WARNING":"This response format is experimental.  It is likely to change in the future.",
  "error":{
    "metadata":[
      "error-class","org.apache.solr.common.SolrException",
      "root-error-class","org.apache.solr.common.SolrException"],
    "msg":"4 out of 5 the property overlay to be of version 22 within 30 seconds! Failed cores: [http://127.0.0.1:63990/collection1_shard2_replica_n1/, http://127.0.0.1:64061/collection1_shard2_replica_n6/, http://127.0.0.1:64027/collection1_shard1_replica_n2/, http://127.0.0.1:64084/collection1_shard1_replica_n5/]",
    "trace":"org.apache.solr.common.SolrException: 4 out of 5 the property overlay to be of version 22 within 30 seconds! Failed cores: [http://127.0.0.1:63990/collection1_shard2_replica_n1/, http://127.0.0.1:64061/collection1_shard2_replica_n6/, http://127.0.0.1:64027/collection1_shard1_replica_n2/, http://127.0.0.1:64084/collection1_shard1_replica_n5/]\n\tat org.apache.solr.handler.SolrConfigHandler.waitForAllReplicasState(SolrConfigHandler.java:892)\n\tat org.apache.solr.handler.SolrConfigHandler$Command.handleCommands(SolrConfigHandler.java:591)\n\tat org.apache.solr.handler.SolrConfigHandler$Command.handlePOST(SolrConfigHandler.java:404)\n\tat org.apache.solr.handler.SolrConfigHandler.handleRequestBody(SolrConfigHandler.java:143)\n\tat org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:207)\n\tat org.apache.solr.core.SolrCore.execute(SolrCore.java:2866)\n\tat org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:887)\n\tat org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:606)\n\tat org.apache.solr.servlet.SolrDispatchFilter.dispatch(SolrDispatchFilter.java:239)\n\tat org.apache.solr.servlet.SolrDispatchFilter.lambda$doFilter$0(SolrDispatchFilter.java:207)\n\tat org.apache.solr.servlet.ServletUtils.traceHttpRequestExecution2(ServletUtils.java:257)\n\tat org.apache.solr.servlet.ServletUtils.rateLimitRequest(ServletUtils.java:227)\n\tat org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:202)\n\tat org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:184)\n\tat org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)\n\tat org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)\n\tat org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:187)\n\tat org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)\n\tat org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)\n\tat org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)\n\tat org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)\n\tat org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1434)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)\n\tat org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)\n\tat org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)\n\tat org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1349)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)\n\tat org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)\n\tat org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)\n\tat org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:713)\n\tat org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)\n\tat org.eclipse.jetty.server.Server.handle(Server.java:516)\n\tat org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:400)\n\tat org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:645)\n\tat org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:392)\n\tat org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:277)\n\tat org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311)\n\tat org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:105)\n\tat org.eclipse.jetty.io.ChannelEndPoint$1.run(ChannelEndPoint.java:104)\n\tat org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)\n\tat org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n",
    "code":500}}
 expected null, but was:<[4 out of 5 the property overlay to be of version 22 within 30 seconds! Failed cores: [http://127.0.0.1:63990/collection1_shard2_replica_n1/, http://127.0.0.1:64061/collection1_shard2_replica_n6/, http://127.0.0.1:64027/collection1_shard1_replica_n2/, http://127.0.0.1:64084/collection1_shard1_replica_n5/]
]>

Stack Trace:
java.lang.AssertionError: {
  "responseHeader":{
    "status":500,
    "QTime":30082},
  "errorMessages":["4 out of 5 the property overlay to be of version 22 within 30 seconds! Failed cores: [http://127.0.0.1:63990/collection1_shard2_replica_n1/, http://127.0.0.1:64061/collection1_shard2_replica_n6/, http://127.0.0.1:64027/collection1_shard1_replica_n2/, http://127.0.0.1:64084/collection1_shard1_replica_n5/]\n"],
  "WARNING":"This response format is experimental.  It is likely to change in the future.",
  "error":{
    "metadata":[
      "error-class","org.apache.solr.common.SolrException",
      "root-error-class","org.apache.solr.common.SolrException"],
    "msg":"4 out of 5 the property overlay to be of version 22 within 30 seconds! Failed cores: [http://127.0.0.1:63990/collection1_shard2_replica_n1/, http://127.0.0.1:64061/collection1_shard2_replica_n6/, http://127.0.0.1:64027/collection1_shard1_replica_n2/, http://127.0.0.1:64084/collection1_shard1_replica_n5/]",
    "trace":"org.apache.solr.common.SolrException: 4 out of 5 the property overlay to be of version 22 within 30 seconds! Failed cores: [http://127.0.0.1:63990/collection1_shard2_replica_n1/, http://127.0.0.1:64061/collection1_shard2_replica_n6/, http://127.0.0.1:64027/collection1_shard1_replica_n2/, http://127.0.0.1:64084/collection1_shard1_replica_n5/]\n\tat org.apache.solr.handler.SolrConfigHandler.waitForAllReplicasState(SolrConfigHandler.java:892)\n\tat org.apache.solr.handler.SolrConfigHandler$Command.handleCommands(SolrConfigHandler.java:591)\n\tat org.apache.solr.handler.SolrConfigHandler$Command.handlePOST(SolrConfigHandler.java:404)\n\tat org.apache.solr.handler.SolrConfigHandler.handleRequestBody(SolrConfigHandler.java:143)\n\tat org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:207)\n\tat org.apache.solr.core.SolrCore.execute(SolrCore.java:2866)\n\tat org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:887)\n\tat org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:606)\n\tat org.apache.solr.servlet.SolrDispatchFilter.dispatch(SolrDispatchFilter.java:239)\n\tat org.apache.solr.servlet.SolrDispatchFilter.lambda$doFilter$0(SolrDispatchFilter.java:207)\n\tat org.apache.solr.servlet.ServletUtils.traceHttpRequestExecution2(ServletUtils.java:257)\n\tat org.apache.solr.servlet.ServletUtils.rateLimitRequest(ServletUtils.java:227)\n\tat org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:202)\n\tat org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:184)\n\tat org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)\n\tat org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)\n\tat org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:187)\n\tat org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)\n\tat org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)\n\tat org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)\n\tat org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)\n\tat org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1434)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)\n\tat org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)\n\tat org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)\n\tat org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1349)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)\n\tat org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)\n\tat org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)\n\tat org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:713)\n\tat org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)\n\tat org.eclipse.jetty.server.Server.handle(Server.java:516)\n\tat org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:400)\n\tat org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:645)\n\tat org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:392)\n\tat org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:277)\n\tat org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311)\n\tat org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:105)\n\tat org.eclipse.jetty.io.ChannelEndPoint$1.run(ChannelEndPoint.java:104)\n\tat org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)\n\tat org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n",
    "code":500}}
 expected null, but was:<[4 out of 5 the property overlay to be of version 22 within 30 seconds! Failed cores: [http://127.0.0.1:63990/collection1_shard2_replica_n1/, http://127.0.0.1:64061/collection1_shard2_replica_n6/, http://127.0.0.1:64027/collection1_shard1_replica_n2/, http://127.0.0.1:64084/collection1_shard1_replica_n5/]
]>
	at __randomizedtesting.SeedInfo.seed([5A0E41650AFF90CB:D25A7EBFA403FD33]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.junit.Assert.failNotNull(Assert.java:756)
	at org.junit.Assert.assertNull(Assert.java:738)
	at org.apache.solr.core.TestSolrConfigHandler.runConfigCommand(TestSolrConfigHandler.java:226)
	at org.apache.solr.handler.TestSolrConfigHandlerCloud.testAdminPath(TestSolrConfigHandlerCloud.java:63)
	at org.apache.solr.handler.TestSolrConfigHandlerCloud.test(TestSolrConfigHandlerCloud.java:52)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:568)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1164)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1135)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base/java.lang.Thread.run(Thread.java:833)


FAILED:  org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=1149, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@17.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:328)
        at app//org.apache.solr.cloud.ZkController.pathExists(ZkController.java:1228)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:73)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getLines(AbstractAnalysisFactory.java:302)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getWordSet(AbstractAnalysisFactory.java:293)
        at app//org.apache.lucene.analysis.en.AbstractWordsFileFilterFactory.inform(AbstractWordsFileFilterFactory.java:88)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:762)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1470)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1319)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1307)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:654)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:188)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:119)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:279)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:342)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:253)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$658/0x00000008011d88b8.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:49)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:249)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1850)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3394)
        at app//org.apache.solr.core.SolrCore$$Lambda$769/0x000000080129e430.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2761)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1216/0x0000000801682920.run(Unknown Source)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=1149, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@17.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:328)
        at app//org.apache.solr.cloud.ZkController.pathExists(ZkController.java:1228)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:73)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getLines(AbstractAnalysisFactory.java:302)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getWordSet(AbstractAnalysisFactory.java:293)
        at app//org.apache.lucene.analysis.en.AbstractWordsFileFilterFactory.inform(AbstractWordsFileFilterFactory.java:88)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:762)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1470)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1319)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1307)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:654)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:188)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:119)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:279)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:342)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:253)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$658/0x00000008011d88b8.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:49)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:249)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1850)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3394)
        at app//org.apache.solr.core.SolrCore$$Lambda$769/0x000000080129e430.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2761)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1216/0x0000000801682920.run(Unknown Source)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
	at __randomizedtesting.SeedInfo.seed([5A0E41650AFF90CB]:0)

[JENKINS] Solr-main-MacOSX (64bit/jdk-17.0.2) - Build # 878 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/878/
Java: 64bit/jdk-17.0.2 -XX:-UseCompressedOops -XX:+UseG1GC

9 tests failed.
FAILED:  org.apache.solr.cloud.TestConfigSetsAPI.testUploadLegacyManagedSchemaFile

Error Message:
java.util.concurrent.TimeoutException: Timed out waiting for all collections to be fully removed.

Stack Trace:
java.util.concurrent.TimeoutException: Timed out waiting for all collections to be fully removed.
	at org.apache.solr.cloud.MiniSolrCloudCluster.deleteAllCollections(MiniSolrCloudCluster.java:661)
	at org.apache.solr.cloud.TestConfigSetsAPI.tearDown(TestConfigSetsAPI.java:133)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:568)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:1004)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base/java.lang.Thread.run(Thread.java:833)


FAILED:  org.apache.solr.cloud.TestConfigSetsAPI.testSingleFileOverwriteV1

Error Message:
java.util.concurrent.TimeoutException: Timed out waiting for all collections to be fully removed.

Stack Trace:
java.util.concurrent.TimeoutException: Timed out waiting for all collections to be fully removed.
	at __randomizedtesting.SeedInfo.seed([8B406B848879871F:2467752CDFC13391]:0)
	at org.apache.solr.cloud.MiniSolrCloudCluster.deleteAllCollections(MiniSolrCloudCluster.java:661)
	at org.apache.solr.cloud.TestConfigSetsAPI.tearDown(TestConfigSetsAPI.java:133)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:568)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:1004)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base/java.lang.Thread.run(Thread.java:833)


FAILED:  org.apache.solr.cloud.TestConfigSetsAPI.testList

Error Message:
java.lang.AssertionError: expected:<1> but was:<3>

Stack Trace:
java.lang.AssertionError: expected:<1> but was:<3>
	at org.junit.Assert.fail(Assert.java:89)
	at org.junit.Assert.failNotEquals(Assert.java:835)
	at org.junit.Assert.assertEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:633)
	at org.apache.solr.cloud.TestConfigSetsAPI.testList(TestConfigSetsAPI.java:1713)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:568)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base/java.lang.Thread.run(Thread.java:833)


FAILED:  org.apache.solr.cloud.TestConfigSetsAPI.testUploadErrors

Error Message:
java.util.concurrent.TimeoutException: Timed out waiting for all collections to be fully removed.

Stack Trace:
java.util.concurrent.TimeoutException: Timed out waiting for all collections to be fully removed.
	at __randomizedtesting.SeedInfo.seed([8B406B848879871F:A50E94A57E411742]:0)
	at org.apache.solr.cloud.MiniSolrCloudCluster.deleteAllCollections(MiniSolrCloudCluster.java:661)
	at org.apache.solr.cloud.TestConfigSetsAPI.tearDown(TestConfigSetsAPI.java:133)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:568)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:1004)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base/java.lang.Thread.run(Thread.java:833)


FAILED:  org.apache.solr.cloud.TestConfigSetsAPI.testSingleFileTrustedV2

Error Message:
java.util.concurrent.TimeoutException: Timed out waiting for all collections to be fully removed.

Stack Trace:
java.util.concurrent.TimeoutException: Timed out waiting for all collections to be fully removed.
	at __randomizedtesting.SeedInfo.seed([8B406B848879871F:5313665BF67BE753]:0)
	at org.apache.solr.cloud.MiniSolrCloudCluster.deleteAllCollections(MiniSolrCloudCluster.java:661)
	at org.apache.solr.cloud.TestConfigSetsAPI.tearDown(TestConfigSetsAPI.java:133)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:568)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:1004)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base/java.lang.Thread.run(Thread.java:833)


FAILED:  org.apache.solr.cloud.TestConfigSetsAPI.testSingleWithCleanupV1

Error Message:
java.util.concurrent.TimeoutException: Timed out waiting for all collections to be fully removed.

Stack Trace:
java.util.concurrent.TimeoutException: Timed out waiting for all collections to be fully removed.
	at __randomizedtesting.SeedInfo.seed([8B406B848879871F:2AF8AA0DA1D84F43]:0)
	at org.apache.solr.cloud.MiniSolrCloudCluster.deleteAllCollections(MiniSolrCloudCluster.java:661)
	at org.apache.solr.cloud.TestConfigSetsAPI.tearDown(TestConfigSetsAPI.java:133)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:568)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:1004)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base/java.lang.Thread.run(Thread.java:833)


FAILED:  org.apache.solr.cloud.TestConfigSetsAPI.testOverwriteWithCleanupV1

Error Message:
java.util.concurrent.TimeoutException: Timed out waiting for all collections to be fully removed.

Stack Trace:
java.util.concurrent.TimeoutException: Timed out waiting for all collections to be fully removed.
	at __randomizedtesting.SeedInfo.seed([8B406B848879871F:6BDE9B37B0B845EC]:0)
	at org.apache.solr.cloud.MiniSolrCloudCluster.deleteAllCollections(MiniSolrCloudCluster.java:661)
	at org.apache.solr.cloud.TestConfigSetsAPI.tearDown(TestConfigSetsAPI.java:133)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:568)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:1004)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base/java.lang.Thread.run(Thread.java:833)


FAILED:  org.apache.solr.cloud.TestConfigSetsAPI.testUpload

Error Message:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:59063/solr: Underlying core creation failed while creating collection: newcollection

Stack Trace:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:59063/solr: Underlying core creation failed while creating collection: newcollection
	at __randomizedtesting.SeedInfo.seed([8B406B848879871F:88FA3B856B1BCF95]:0)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:715)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:405)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:371)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1248)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:969)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:896)
	at app//org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1193)
	at app//org.apache.solr.cloud.TestConfigSetsAPI.createCollection(TestConfigSetsAPI.java:1556)
	at app//org.apache.solr.cloud.TestConfigSetsAPI.testUpload(TestConfigSetsAPI.java:1169)
	at java.base@17.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base@17.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base@17.0.2/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base@17.0.2/java.lang.reflect.Method.invoke(Method.java:568)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)


FAILED:  org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=1064, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@17.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:328)
        at app//org.apache.solr.cloud.ZkController.pathExists(ZkController.java:1228)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:73)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getLines(AbstractAnalysisFactory.java:302)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getWordSet(AbstractAnalysisFactory.java:293)
        at app//org.apache.lucene.analysis.en.AbstractWordsFileFilterFactory.inform(AbstractWordsFileFilterFactory.java:88)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:762)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1470)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1319)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1307)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:654)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:188)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:119)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:279)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:342)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:253)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$633/0x0000000801159138.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:49)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:249)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1850)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3394)
        at app//org.apache.solr.core.SolrCore$$Lambda$743/0x000000080121b440.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2761)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1196/0x000000080163ae10.run(Unknown Source)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=1064, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@17.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:328)
        at app//org.apache.solr.cloud.ZkController.pathExists(ZkController.java:1228)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:73)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getLines(AbstractAnalysisFactory.java:302)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getWordSet(AbstractAnalysisFactory.java:293)
        at app//org.apache.lucene.analysis.en.AbstractWordsFileFilterFactory.inform(AbstractWordsFileFilterFactory.java:88)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:762)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1470)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1319)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1307)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:654)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:188)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:119)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:279)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:342)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:253)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$633/0x0000000801159138.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:49)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:249)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1850)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3394)
        at app//org.apache.solr.core.SolrCore$$Lambda$743/0x000000080121b440.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2761)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1196/0x000000080163ae10.run(Unknown Source)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
	at __randomizedtesting.SeedInfo.seed([8B406B848879871F]:0)

[JENKINS] Solr-main-MacOSX (64bit/jdk-16.0.2) - Build # 877 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/877/
Java: 64bit/jdk-16.0.2 -XX:-UseCompressedOops -XX:+UseG1GC

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test

Error Message:
java.lang.AssertionError: Didn't see replicas [core_node6, core_node3] come up within 90000 ms! ClusterState: DocCollection(c8n_1x3_lf//collections/c8n_1x3_lf/state.json/15 )={
  "pullReplicas":"0",
  "configName":"conf1",
  "replicationFactor":"3",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node3":{
          "core":"c8n_1x3_lf_shard1_replica_n1",
          "leader":"true",
          "core_node_name":"core_node3",
          "node_name":"127.0.0.1:60473_akzn",
          "base_url":"http://127.0.0.1:60473/akzn",
          "state":"active",
          "collection":"c8n_1x3_lf",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node5":{
          "core":"c8n_1x3_lf_shard1_replica_n2",
          "core_node_name":"core_node5",
          "node_name":"127.0.0.1:60514_akzn",
          "base_url":"http://127.0.0.1:60514/akzn",
          "state":"down",
          "collection":"c8n_1x3_lf",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node6":{
          "core":"c8n_1x3_lf_shard1_replica_n4",
          "core_node_name":"core_node6",
          "node_name":"127.0.0.1:60520_akzn",
          "base_url":"http://127.0.0.1:60520/akzn",
          "state":"recovering",
          "collection":"c8n_1x3_lf",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"3",
  "tlogReplicas":"0"}

Stack Trace:
java.lang.AssertionError: Didn't see replicas [core_node6, core_node3] come up within 90000 ms! ClusterState: DocCollection(c8n_1x3_lf//collections/c8n_1x3_lf/state.json/15 )={
  "pullReplicas":"0",
  "configName":"conf1",
  "replicationFactor":"3",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node3":{
          "core":"c8n_1x3_lf_shard1_replica_n1",
          "leader":"true",
          "core_node_name":"core_node3",
          "node_name":"127.0.0.1:60473_akzn",
          "base_url":"http://127.0.0.1:60473/akzn",
          "state":"active",
          "collection":"c8n_1x3_lf",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node5":{
          "core":"c8n_1x3_lf_shard1_replica_n2",
          "core_node_name":"core_node5",
          "node_name":"127.0.0.1:60514_akzn",
          "base_url":"http://127.0.0.1:60514/akzn",
          "state":"down",
          "collection":"c8n_1x3_lf",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node6":{
          "core":"c8n_1x3_lf_shard1_replica_n4",
          "core_node_name":"core_node6",
          "node_name":"127.0.0.1:60520_akzn",
          "base_url":"http://127.0.0.1:60520/akzn",
          "state":"recovering",
          "collection":"c8n_1x3_lf",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"3",
  "tlogReplicas":"0"}
	at __randomizedtesting.SeedInfo.seed([1F34B52AD5FC8319:97608AF07B00EEE1]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.solr.cloud.HttpPartitionTest.waitToSeeReplicasActive(HttpPartitionTest.java:687)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:198)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:56)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:78)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:567)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1164)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1135)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base/java.lang.Thread.run(Thread.java:831)


FAILED:  org.apache.solr.cloud.api.collections.ShardSplitTest.testSplitShardWithRule

Error Message:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:53186/f_nzs: Underlying core creation failed while creating collection: shardSplitWithRule_rewrite

Stack Trace:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:53186/f_nzs: Underlying core creation failed while creating collection: shardSplitWithRule_rewrite
	at __randomizedtesting.SeedInfo.seed([1F34B52AD5FC8319:9E3A6498480ED832]:0)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:715)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:405)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:371)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1248)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:969)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:896)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:223)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:238)
	at app//org.apache.solr.cloud.api.collections.ShardSplitTest.doSplitShardWithRule(ShardSplitTest.java:754)
	at app//org.apache.solr.cloud.api.collections.ShardSplitTest.testSplitShardWithRule(ShardSplitTest.java:737)
	at java.base@16.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base@16.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:78)
	at java.base@16.0.2/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base@16.0.2/java.lang.reflect.Method.invoke(Method.java:567)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1164)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1135)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)


FAILED:  org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=1182, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:328)
        at app//org.apache.solr.cloud.ZkController.pathExists(ZkController.java:1228)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:73)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getLines(AbstractAnalysisFactory.java:302)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getWordSet(AbstractAnalysisFactory.java:293)
        at app//org.apache.lucene.analysis.en.AbstractWordsFileFilterFactory.inform(AbstractWordsFileFilterFactory.java:88)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:762)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1470)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1319)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1307)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:654)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:188)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:119)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:279)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:342)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:253)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$629/0x000000080114b000.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:49)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:249)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1850)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3394)
        at app//org.apache.solr.core.SolrCore$$Lambda$739/0x0000000801211040.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2761)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1207/0x000000080168fd08.run(Unknown Source)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=1182, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:328)
        at app//org.apache.solr.cloud.ZkController.pathExists(ZkController.java:1228)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:73)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getLines(AbstractAnalysisFactory.java:302)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getWordSet(AbstractAnalysisFactory.java:293)
        at app//org.apache.lucene.analysis.en.AbstractWordsFileFilterFactory.inform(AbstractWordsFileFilterFactory.java:88)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:762)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1470)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1319)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1307)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:654)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:188)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:119)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:279)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:342)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:253)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$629/0x000000080114b000.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:49)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:249)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1850)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3394)
        at app//org.apache.solr.core.SolrCore$$Lambda$739/0x0000000801211040.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2761)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1207/0x000000080168fd08.run(Unknown Source)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
	at __randomizedtesting.SeedInfo.seed([1F34B52AD5FC8319]:0)

[JENKINS] Solr-main-MacOSX (64bit/jdk-17.0.2) - Build # 876 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/876/
Java: 64bit/jdk-17.0.2 -XX:+UseCompressedOops -XX:+UseSerialGC

6 tests failed.
FAILED:  org.apache.solr.cloud.DistribJoinFromCollectionTest.testNoScore

Error Message:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:49408/solr/to_2x2: Error from server at null: SolrCloud join: Collection 'from_1x4Alias' not found!

Stack Trace:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:49408/solr/to_2x2: Error from server at null: SolrCloud join: Collection 'from_1x4Alias' not found!
	at __randomizedtesting.SeedInfo.seed([FF08137E2985E585:40871FCCD7C9FAC9]:0)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:715)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:405)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:371)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1248)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:969)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:896)
	at app//org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1193)
	at app//org.apache.solr.cloud.DistribJoinFromCollectionTest.testJoins(DistribJoinFromCollectionTest.java:176)
	at app//org.apache.solr.cloud.DistribJoinFromCollectionTest.testNoScore(DistribJoinFromCollectionTest.java:111)
	at java.base@17.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base@17.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base@17.0.2/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base@17.0.2/java.lang.reflect.Method.invoke(Method.java:568)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)


FAILED:  org.apache.solr.cluster.events.ClusterEventProducerTest.testListenerPlugins

Error Message:
java.lang.AssertionError: timestamp of the event is in the future

Stack Trace:
java.lang.AssertionError: timestamp of the event is in the future
	at __randomizedtesting.SeedInfo.seed([FF08137E2985E585:73D7507F8FE99077]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.junit.Assert.assertTrue(Assert.java:42)
	at org.apache.solr.cluster.events.ClusterEventProducerTest.testListenerPlugins(ClusterEventProducerTest.java:405)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:568)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base/java.lang.Thread.run(Thread.java:833)


FAILED:  org.apache.solr.schema.PreAnalyzedFieldManagedSchemaCloudTest.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=3862, name=qtp45314725-3862, state=RUNNABLE, group=TGRP-PreAnalyzedFieldManagedSchemaCloudTest], registration stack trace below.

Stack Trace:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=3862, name=qtp45314725-3862, state=RUNNABLE, group=TGRP-PreAnalyzedFieldManagedSchemaCloudTest], registration stack trace below.
	at java.base@17.0.2/java.lang.Thread.getStackTrace(Thread.java:1610)
	at app//com.carrotsearch.randomizedtesting.RandomizedContext.closeAtEnd(RandomizedContext.java:157)
	at app//org.apache.lucene.util.LuceneTestCase.closeAfterSuite(LuceneTestCase.java:798)
	at app//org.apache.lucene.util.LuceneTestCase.wrapDirectory(LuceneTestCase.java:1498)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1368)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1350)
	at app//org.apache.solr.core.MockDirectoryFactory.create(MockDirectoryFactory.java:51)
	at app//org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:368)
	at app//org.apache.solr.core.SolrCore.initSnapshotMetaDataManager(SolrCore.java:606)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1119)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1048)
	at app//org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1562)
	at app//org.apache.solr.core.CoreContainer.create(CoreContainer.java:1428)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:389)
	at app//org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:422)
	at app//org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:205)
	at app//org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:207)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:941)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:893)
	at app//org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:584)
	at app//org.apache.solr.servlet.SolrDispatchFilter.dispatch(SolrDispatchFilter.java:239)
	at app//org.apache.solr.servlet.SolrDispatchFilter.lambda$doFilter$0(SolrDispatchFilter.java:207)
	at app//org.apache.solr.servlet.ServletUtils.traceHttpRequestExecution2(ServletUtils.java:257)
	at app//org.apache.solr.servlet.ServletUtils.rateLimitRequest(ServletUtils.java:227)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:202)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:184)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:187)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1434)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
	at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)
	at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1349)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
	at app//org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:713)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.server.Server.handle(Server.java:516)
	at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:400)
	at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:645)
	at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:392)
	at app//org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:352)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
	at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
Caused by: java.lang.AssertionError: Directory not closed: RawDirectoryWrapper(ByteBuffersDirectory@8125479 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@153edc80)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.lucene.util.CloseableDirectory.close(CloseableDirectory.java:43)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:789)
	at com.carrotsearch.randomizedtesting.RandomizedContext.closeResources(RandomizedContext.java:225)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.afterAlways(RandomizedRunner.java:805)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	... 1 more


FAILED:  org.apache.solr.schema.PreAnalyzedFieldManagedSchemaCloudTest.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=3862, name=qtp45314725-3862, state=RUNNABLE, group=TGRP-PreAnalyzedFieldManagedSchemaCloudTest], registration stack trace below.

Stack Trace:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=3862, name=qtp45314725-3862, state=RUNNABLE, group=TGRP-PreAnalyzedFieldManagedSchemaCloudTest], registration stack trace below.
	at java.base@17.0.2/java.lang.Thread.getStackTrace(Thread.java:1610)
	at app//com.carrotsearch.randomizedtesting.RandomizedContext.closeAtEnd(RandomizedContext.java:157)
	at app//org.apache.lucene.util.LuceneTestCase.closeAfterSuite(LuceneTestCase.java:798)
	at app//org.apache.lucene.util.LuceneTestCase.wrapDirectory(LuceneTestCase.java:1504)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1368)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1350)
	at app//org.apache.solr.core.MockDirectoryFactory.create(MockDirectoryFactory.java:51)
	at app//org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:368)
	at app//org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:412)
	at app//org.apache.solr.core.SolrCore.initIndex(SolrCore.java:856)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1123)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1048)
	at app//org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1562)
	at app//org.apache.solr.core.CoreContainer.create(CoreContainer.java:1428)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:389)
	at app//org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:422)
	at app//org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:205)
	at app//org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:207)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:941)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:893)
	at app//org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:584)
	at app//org.apache.solr.servlet.SolrDispatchFilter.dispatch(SolrDispatchFilter.java:239)
	at app//org.apache.solr.servlet.SolrDispatchFilter.lambda$doFilter$0(SolrDispatchFilter.java:207)
	at app//org.apache.solr.servlet.ServletUtils.traceHttpRequestExecution2(ServletUtils.java:257)
	at app//org.apache.solr.servlet.ServletUtils.rateLimitRequest(ServletUtils.java:227)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:202)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:184)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:187)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1434)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
	at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)
	at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1349)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
	at app//org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:713)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.server.Server.handle(Server.java:516)
	at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:400)
	at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:645)
	at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:392)
	at app//org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:352)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
	at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
Caused by: java.lang.AssertionError: Directory not closed: MockDirectoryWrapper(ByteBuffersDirectory@7c2f21fd lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7020e28d)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.lucene.util.CloseableDirectory.close(CloseableDirectory.java:43)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:789)
	at com.carrotsearch.randomizedtesting.RandomizedContext.closeResources(RandomizedContext.java:225)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.afterAlways(RandomizedRunner.java:805)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	... 1 more


FAILED:  org.apache.solr.schema.PreAnalyzedFieldManagedSchemaCloudTest.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=3862, name=qtp45314725-3862, state=RUNNABLE, group=TGRP-PreAnalyzedFieldManagedSchemaCloudTest], registration stack trace below.

Stack Trace:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=3862, name=qtp45314725-3862, state=RUNNABLE, group=TGRP-PreAnalyzedFieldManagedSchemaCloudTest], registration stack trace below.
	at java.base@17.0.2/java.lang.Thread.getStackTrace(Thread.java:1610)
	at app//com.carrotsearch.randomizedtesting.RandomizedContext.closeAtEnd(RandomizedContext.java:157)
	at app//org.apache.lucene.util.LuceneTestCase.closeAfterSuite(LuceneTestCase.java:798)
	at app//org.apache.lucene.util.LuceneTestCase.wrapDirectory(LuceneTestCase.java:1504)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1368)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1350)
	at app//org.apache.solr.core.MockDirectoryFactory.create(MockDirectoryFactory.java:51)
	at app//org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:368)
	at app//org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:108)
	at app//org.apache.solr.core.SolrCore.initIndex(SolrCore.java:895)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1123)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1048)
	at app//org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1562)
	at app//org.apache.solr.core.CoreContainer.create(CoreContainer.java:1428)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:389)
	at app//org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:422)
	at app//org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:205)
	at app//org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:207)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:941)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:893)
	at app//org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:584)
	at app//org.apache.solr.servlet.SolrDispatchFilter.dispatch(SolrDispatchFilter.java:239)
	at app//org.apache.solr.servlet.SolrDispatchFilter.lambda$doFilter$0(SolrDispatchFilter.java:207)
	at app//org.apache.solr.servlet.ServletUtils.traceHttpRequestExecution2(ServletUtils.java:257)
	at app//org.apache.solr.servlet.ServletUtils.rateLimitRequest(ServletUtils.java:227)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:202)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:184)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:187)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1434)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
	at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)
	at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1349)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
	at app//org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:713)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.server.Server.handle(Server.java:516)
	at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:400)
	at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:645)
	at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:392)
	at app//org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:352)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
	at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
Caused by: java.lang.AssertionError: Directory not closed: MockDirectoryWrapper(ByteBuffersDirectory@7d08e318 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@55ceaa17)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.lucene.util.CloseableDirectory.close(CloseableDirectory.java:43)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:789)
	at com.carrotsearch.randomizedtesting.RandomizedContext.closeResources(RandomizedContext.java:225)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.afterAlways(RandomizedRunner.java:805)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	... 1 more


FAILED:  org.apache.solr.schema.PreAnalyzedFieldManagedSchemaCloudTest.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.schema.PreAnalyzedFieldManagedSchemaCloudTest: 
   1) Thread[id=3921, name=ZKEventListenerThread, state=TERMINATED, group={null group}]
        at java.base@17.0.2/java.util.concurrent.ConcurrentHashMap.clear(ConcurrentHashMap.java:1213)
        at app//org.apache.solr.core.SolrCore.doClose(SolrCore.java:1840)
        at app//org.apache.solr.core.SolrCore.close(SolrCore.java:1736)
        at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1196)
        at app//org.apache.solr.core.SolrCore.reload(SolrCore.java:780)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1857)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3394)
        at app//org.apache.solr.core.SolrCore$$Lambda$746/0x000000080121c000.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2761)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1219/0x0000000801695450.run(Unknown Source)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   2) Thread[id=3940, name=searcherExecutor-3420-thread-1, state=TERMINATED, group={null group}]
        at (empty stack)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.schema.PreAnalyzedFieldManagedSchemaCloudTest: 
   1) Thread[id=3921, name=ZKEventListenerThread, state=TERMINATED, group={null group}]
        at java.base@17.0.2/java.util.concurrent.ConcurrentHashMap.clear(ConcurrentHashMap.java:1213)
        at app//org.apache.solr.core.SolrCore.doClose(SolrCore.java:1840)
        at app//org.apache.solr.core.SolrCore.close(SolrCore.java:1736)
        at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1196)
        at app//org.apache.solr.core.SolrCore.reload(SolrCore.java:780)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1857)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3394)
        at app//org.apache.solr.core.SolrCore$$Lambda$746/0x000000080121c000.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2761)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1219/0x0000000801695450.run(Unknown Source)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   2) Thread[id=3940, name=searcherExecutor-3420-thread-1, state=TERMINATED, group={null group}]
        at (empty stack)
	at __randomizedtesting.SeedInfo.seed([FF08137E2985E585]:0)

[JENKINS] Solr-main-MacOSX (64bit/jdk-17.0.2) - Build # 875 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/875/
Java: 64bit/jdk-17.0.2 -XX:+UseCompressedOops -XX:+UseParallelGC

4 tests failed.
FAILED:  org.apache.solr.cloud.HttpPartitionOnCommitTest.test

Error Message:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:59009/tumh: Underlying core creation failed while creating collection: control_collection

Stack Trace:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:59009/tumh: Underlying core creation failed while creating collection: control_collection
	at __randomizedtesting.SeedInfo.seed([459D5AE4E2B11562:CDC9653E4C4D789A]:0)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:715)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:405)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:371)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1248)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:969)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:896)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:223)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:238)
	at app//org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:357)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1162)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1135)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)


FAILED:  org.apache.solr.cloud.api.collections.CollectionsAPIDistributedZkTest.deletePartiallyCreatedCollection

Error Message:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:59187/solr: ADDREPLICA failed to create replica

Stack Trace:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:59187/solr: ADDREPLICA failed to create replica
	at __randomizedtesting.SeedInfo.seed([459D5AE4E2B11562:6F1D3601556B35AA]:0)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:715)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:405)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:371)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1248)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:969)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:896)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:223)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:238)
	at app//org.apache.solr.cloud.api.collections.AbstractCollectionsAPIDistributedZkTestBase.deletePartiallyCreatedCollection(AbstractCollectionsAPIDistributedZkTestBase.java:167)
	at java.base@17.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base@17.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base@17.0.2/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base@17.0.2/java.lang.reflect.Method.invoke(Method.java:568)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)


FAILED:  org.apache.solr.common.cloud.PerReplicaStatesIntegrationTest.testRestart

Error Message:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:51930/solr: org.apache.solr.common.SolrException
	at org.apache.solr.client.solrj.SolrResponse.getException(SolrResponse.java:61)
	at org.apache.solr.handler.admin.CollectionsHandler.invokeAction(CollectionsHandler.java:263)
	at org.apache.solr.handler.admin.CollectionsHandler.handleRequestBody(CollectionsHandler.java:212)
	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:207)
	at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:941)
	at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:893)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:584)
	at org.apache.solr.servlet.SolrDispatchFilter.dispatch(SolrDispatchFilter.java:239)
	at org.apache.solr.servlet.SolrDispatchFilter.lambda$doFilter$0(SolrDispatchFilter.java:207)
	at org.apache.solr.servlet.ServletUtils.traceHttpRequestExecution2(ServletUtils.java:257)
	at org.apache.solr.servlet.ServletUtils.rateLimitRequest(ServletUtils.java:227)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:202)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:184)
	at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:187)
	at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1434)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1349)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:763)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at org.eclipse.jetty.server.Server.handle(Server.java:516)
	at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:400)
	at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:645)
	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:392)
	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:277)
	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311)
	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:105)
	at org.eclipse.jetty.io.ChannelEndPoint$1.run(ChannelEndPoint.java:104)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
	at java.base/java.lang.Thread.run(Thread.java:833)


Stack Trace:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:51930/solr: org.apache.solr.common.SolrException
	at org.apache.solr.client.solrj.SolrResponse.getException(SolrResponse.java:61)
	at org.apache.solr.handler.admin.CollectionsHandler.invokeAction(CollectionsHandler.java:263)
	at org.apache.solr.handler.admin.CollectionsHandler.handleRequestBody(CollectionsHandler.java:212)
	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:207)
	at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:941)
	at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:893)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:584)
	at org.apache.solr.servlet.SolrDispatchFilter.dispatch(SolrDispatchFilter.java:239)
	at org.apache.solr.servlet.SolrDispatchFilter.lambda$doFilter$0(SolrDispatchFilter.java:207)
	at org.apache.solr.servlet.ServletUtils.traceHttpRequestExecution2(ServletUtils.java:257)
	at org.apache.solr.servlet.ServletUtils.rateLimitRequest(ServletUtils.java:227)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:202)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:184)
	at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:187)
	at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1434)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1349)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:763)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at org.eclipse.jetty.server.Server.handle(Server.java:516)
	at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:400)
	at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:645)
	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:392)
	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:277)
	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311)
	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:105)
	at org.eclipse.jetty.io.ChannelEndPoint$1.run(ChannelEndPoint.java:104)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
	at java.base/java.lang.Thread.run(Thread.java:833)

	at __randomizedtesting.SeedInfo.seed([459D5AE4E2B11562:D010B6856759A177]:0)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:715)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:405)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:371)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1248)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:969)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:896)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:223)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:238)
	at app//org.apache.solr.common.cloud.PerReplicaStatesIntegrationTest.testRestart(PerReplicaStatesIntegrationTest.java:123)
	at java.base@17.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base@17.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base@17.0.2/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base@17.0.2/java.lang.reflect.Method.invoke(Method.java:568)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)


FAILED:  org.apache.solr.cloud.TestStressLiveNodes.testStress

Error Message:
java.lang.AssertionError: iter143 expected:<3> but was:<4>

Stack Trace:
java.lang.AssertionError: iter143 expected:<3> but was:<4>
	at __randomizedtesting.SeedInfo.seed([459D5AE4E2B11562:508627256146ED19]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.junit.Assert.failNotEquals(Assert.java:835)
	at org.junit.Assert.assertEquals(Assert.java:647)
	at org.apache.solr.cloud.TestStressLiveNodes.testStress(TestStressLiveNodes.java:211)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:568)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base/java.lang.Thread.run(Thread.java:833)

[JENKINS] Solr-main-MacOSX (64bit/jdk-16.0.2) - Build # 874 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/874/
Java: 64bit/jdk-16.0.2 -XX:-UseCompressedOops -XX:+UseSerialGC

4 tests failed.
FAILED:  org.apache.solr.cloud.HttpPartitionWithTlogReplicasTest.test

Error Message:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:53401/ec_pv/e: Underlying core creation failed while creating collection: c8n_1x2_leader_session_loss

Stack Trace:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:53401/ec_pv/e: Underlying core creation failed while creating collection: c8n_1x2_leader_session_loss
	at __randomizedtesting.SeedInfo.seed([58C05331E518725E:D0946CEB4BE41FA6]:0)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:715)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at app//org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1193)
	at app//org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:2099)
	at app//org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:2137)
	at app//org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:2025)
	at app//org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollectionRetry(AbstractFullDistribZkTestBase.java:2404)
	at app//org.apache.solr.cloud.HttpPartitionTest.testLeaderZkSessionLoss(HttpPartitionTest.java:421)
	at app//org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:149)
	at java.base@16.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base@16.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:78)
	at java.base@16.0.2/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base@16.0.2/java.lang.reflect.Method.invoke(Method.java:567)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1164)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1135)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)


FAILED:  org.apache.solr.cloud.TestStressLiveNodes.testStress

Error Message:
java.lang.AssertionError: iter15 expected:<15> but was:<17>

Stack Trace:
java.lang.AssertionError: iter15 expected:<15> but was:<17>
	at __randomizedtesting.SeedInfo.seed([58C05331E518725E:4DDB2EF066EF8A25]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.junit.Assert.failNotEquals(Assert.java:835)
	at org.junit.Assert.assertEquals(Assert.java:647)
	at org.apache.solr.cloud.TestStressLiveNodes.testStress(TestStressLiveNodes.java:211)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:78)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:567)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base/java.lang.Thread.run(Thread.java:831)


FAILED:  org.apache.solr.cloud.api.collections.CollectionsAPIDistributedZkTest.testCollectionsAPI

Error Message:
java.lang.RuntimeException: Failed while waiting for active collection
Timeout waiting to see state for collection=awhollynewcollection_0 :DocCollection(awhollynewcollection_0//collections/awhollynewcollection_0/state.json/20 )={
  "pullReplicas":"0",
  "configName":"conf2",
  "replicationFactor":"4",
  "shards":{
    "shard1":{
      "range":"80000000-b332ffff",
      "state":"active",
      "replicas":{
        "core_node11":{
          "core":"awhollynewcollection_0_shard1_replica_n8",
          "leader":"true",
          "core_node_name":"core_node11",
          "node_name":"127.0.0.1:59391_solr",
          "base_url":"http://127.0.0.1:59391/solr",
          "state":"active",
          "collection":"awhollynewcollection_0",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node19":{
          "core":"awhollynewcollection_0_shard1_replica_n16",
          "core_node_name":"core_node19",
          "node_name":"127.0.0.1:59384_solr",
          "base_url":"http://127.0.0.1:59384/solr",
          "state":"active",
          "collection":"awhollynewcollection_0",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node21":{
          "core":"awhollynewcollection_0_shard1_replica_n18",
          "core_node_name":"core_node21",
          "node_name":"127.0.0.1:59390_solr",
          "base_url":"http://127.0.0.1:59390/solr",
          "state":"active",
          "collection":"awhollynewcollection_0",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node37":{
          "core":"awhollynewcollection_0_shard1_replica_n34",
          "core_node_name":"core_node37",
          "node_name":"127.0.0.1:59386_solr",
          "base_url":"http://127.0.0.1:59386/solr",
          "state":"active",
          "collection":"awhollynewcollection_0",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"}}},
    "shard2":{
      "range":"b3330000-e665ffff",
      "state":"active",
      "replicas":{
        "core_node7":{
          "core":"awhollynewcollection_0_shard2_replica_n4",
          "core_node_name":"core_node7",
          "node_name":"127.0.0.1:59384_solr",
          "base_url":"http://127.0.0.1:59384/solr",
          "state":"active",
          "collection":"awhollynewcollection_0",
          "shard":"shard2",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node9":{
          "core":"awhollynewcollection_0_shard2_replica_n6",
          "core_node_name":"core_node9",
          "node_name":"127.0.0.1:59386_solr",
          "base_url":"http://127.0.0.1:59386/solr",
          "state":"active",
          "collection":"awhollynewcollection_0",
          "shard":"shard2",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node15":{
          "core":"awhollynewcollection_0_shard2_replica_n12",
          "core_node_name":"core_node15",
          "node_name":"127.0.0.1:59390_solr",
          "base_url":"http://127.0.0.1:59390/solr",
          "state":"active",
          "collection":"awhollynewcollection_0",
          "shard":"shard2",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node29":{
          "core":"awhollynewcollection_0_shard2_replica_n26",
          "leader":"true",
          "core_node_name":"core_node29",
          "node_name":"127.0.0.1:59391_solr",
          "base_url":"http://127.0.0.1:59391/solr",
          "state":"active",
          "collection":"awhollynewcollection_0",
          "shard":"shard2",
          "type":"NRT",
          "force_set_state":"false"}}},
    "shard3":{
      "range":"e6660000-1998ffff",
      "state":"active",
      "replicas":{
        "core_node5":{
          "core":"awhollynewcollection_0_shard3_replica_n2",
          "core_node_name":"core_node5",
          "node_name":"127.0.0.1:59386_solr",
          "base_url":"http://127.0.0.1:59386/solr",
          "state":"active",
          "collection":"awhollynewcollection_0",
          "shard":"shard3",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node17":{
          "core":"awhollynewcollection_0_shard3_replica_n14",
          "core_node_name":"core_node17",
          "node_name":"127.0.0.1:59390_solr",
          "base_url":"http://127.0.0.1:59390/solr",
          "state":"active",
          "collection":"awhollynewcollection_0",
          "shard":"shard3",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node24":{
          "core":"awhollynewcollection_0_shard3_replica_n22",
          "leader":"true",
          "core_node_name":"core_node24",
          "node_name":"127.0.0.1:59391_solr",
          "base_url":"http://127.0.0.1:59391/solr",
          "state":"active",
          "collection":"awhollynewcollection_0",
          "shard":"shard3",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node33":{
          "core":"awhollynewcollection_0_shard3_replica_n31",
          "core_node_name":"core_node33",
          "node_name":"127.0.0.1:59384_solr",
          "base_url":"http://127.0.0.1:59384/solr",
          "state":"active",
          "collection":"awhollynewcollection_0",
          "shard":"shard3",
          "type":"NRT",
          "force_set_state":"false"}}},
    "shard4":{
      "range":"19990000-4ccbffff",
      "state":"active",
      "replicas":{
        "core_node30":{
          "core":"awhollynewcollection_0_shard4_replica_n28",
          "core_node_name":"core_node30",
          "node_name":"127.0.0.1:59386_solr",
          "base_url":"http://127.0.0.1:59386/solr",
          "state":"down",
          "collection":"awhollynewcollection_0",
          "shard":"shard4",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node35":{
          "core":"awhollynewcollection_0_shard4_replica_n32",
          "core_node_name":"core_node35",
          "node_name":"127.0.0.1:59384_solr",
          "base_url":"http://127.0.0.1:59384/solr",
          "state":"down",
          "collection":"awhollynewcollection_0",
          "shard":"shard4",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node39":{
          "core":"awhollynewcollection_0_shard4_replica_n36",
          "core_node_name":"core_node39",
          "node_name":"127.0.0.1:59390_solr",
          "base_url":"http://127.0.0.1:59390/solr",
          "state":"down",
          "collection":"awhollynewcollection_0",
          "shard":"shard4",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node40":{
          "core":"awhollynewcollection_0_shard4_replica_n38",
          "leader":"true",
          "core_node_name":"core_node40",
          "node_name":"127.0.0.1:59391_solr",
          "base_url":"http://127.0.0.1:59391/solr",
          "state":"down",
          "collection":"awhollynewcollection_0",
          "shard":"shard4",
          "type":"NRT",
          "force_set_state":"false"}}},
    "shard5":{
      "range":"4ccc0000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node3":{
          "core":"awhollynewcollection_0_shard5_replica_n1",
          "leader":"true",
          "core_node_name":"core_node3",
          "node_name":"127.0.0.1:59391_solr",
          "base_url":"http://127.0.0.1:59391/solr",
          "state":"active",
          "collection":"awhollynewcollection_0",
          "shard":"shard5",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node13":{
          "core":"awhollynewcollection_0_shard5_replica_n10",
          "core_node_name":"core_node13",
          "node_name":"127.0.0.1:59384_solr",
          "base_url":"http://127.0.0.1:59384/solr",
          "state":"active",
          "collection":"awhollynewcollection_0",
          "shard":"shard5",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node23":{
          "core":"awhollynewcollection_0_shard5_replica_n20",
          "core_node_name":"core_node23",
          "node_name":"127.0.0.1:59386_solr",
          "base_url":"http://127.0.0.1:59386/solr",
          "state":"active",
          "collection":"awhollynewcollection_0",
          "shard":"shard5",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node27":{
          "core":"awhollynewcollection_0_shard5_replica_n25",
          "core_node_name":"core_node27",
          "node_name":"127.0.0.1:59390_solr",
          "base_url":"http://127.0.0.1:59390/solr",
          "state":"active",
          "collection":"awhollynewcollection_0",
          "shard":"shard5",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"4",
  "tlogReplicas":"0"}
Live Nodes: [127.0.0.1:59384_solr, 127.0.0.1:59386_solr, 127.0.0.1:59390_solr, 127.0.0.1:59391_solr]
Last available state: DocCollection(awhollynewcollection_0//collections/awhollynewcollection_0/state.json/20 )={
  "pullReplicas":"0",
  "configName":"conf2",
  "replicationFactor":"4",
  "shards":{
    "shard1":{
      "range":"80000000-b332ffff",
      "state":"active",
      "replicas":{
        "core_node11":{
          "core":"awhollynewcollection_0_shard1_replica_n8",
          "leader":"true",
          "core_node_name":"core_node11",
          "node_name":"127.0.0.1:59391_solr",
          "base_url":"http://127.0.0.1:59391/solr",
          "state":"active",
          "collection":"awhollynewcollection_0",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node19":{
          "core":"awhollynewcollection_0_shard1_replica_n16",
          "core_node_name":"core_node19",
          "node_name":"127.0.0.1:59384_solr",
          "base_url":"http://127.0.0.1:59384/solr",
          "state":"active",
          "collection":"awhollynewcollection_0",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node21":{
          "core":"awhollynewcollection_0_shard1_replica_n18",
          "core_node_name":"core_node21",
          "node_name":"127.0.0.1:59390_solr",
          "base_url":"http://127.0.0.1:59390/solr",
          "state":"active",
          "collection":"awhollynewcollection_0",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node37":{
          "core":"awhollynewcollection_0_shard1_replica_n34",
          "core_node_name":"core_node37",
          "node_name":"127.0.0.1:59386_solr",
          "base_url":"http://127.0.0.1:59386/solr",
          "state":"active",
          "collection":"awhollynewcollection_0",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"}}},
    "shard2":{
      "range":"b3330000-e665ffff",
      "state":"active",
      "replicas":{
        "core_node7":{
          "core":"awhollynewcollection_0_shard2_replica_n4",
          "core_node_name":"core_node7",
          "node_name":"127.0.0.1:59384_solr",
          "base_url":"http://127.0.0.1:59384/solr",
          "state":"active",
          "collection":"awhollynewcollection_0",
          "shard":"shard2",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node9":{
          "core":"awhollynewcollection_0_shard2_replica_n6",
          "core_node_name":"core_node9",
          "node_name":"127.0.0.1:59386_solr",
          "base_url":"http://127.0.0.1:59386/solr",
          "state":"active",
          "collection":"awhollynewcollection_0",
          "shard":"shard2",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node15":{
          "core":"awhollynewcollection_0_shard2_replica_n12",
          "core_node_name":"core_node15",
          "node_name":"127.0.0.1:59390_solr",
          "base_url":"http://127.0.0.1:59390/solr",
          "state":"active",
          "collection":"awhollynewcollection_0",
          "shard":"shard2",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node29":{
          "core":"awhollynewcollection_0_shard2_replica_n26",
          "leader":"true",
          "core_node_name":"core_node29",
          "node_name":"127.0.0.1:59391_solr",
          "base_url":"http://127.0.0.1:59391/solr",
          "state":"active",
          "collection":"awhollynewcollection_0",
          "shard":"shard2",
          "type":"NRT",
          "force_set_state":"false"}}},
    "shard3":{
      "range":"e6660000-1998ffff",
      "state":"active",
      "replicas":{
        "core_node5":{
          "core":"awhollynewcollection_0_shard3_replica_n2",
          "core_node_name":"core_node5",
          "node_name":"127.0.0.1:59386_solr",
          "base_url":"http://127.0.0.1:59386/solr",
          "state":"active",
          "collection":"awhollynewcollection_0",
          "shard":"shard3",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node17":{
          "core":"awhollynewcollection_0_shard3_replica_n14",
          "core_node_name":"core_node17",
          "node_name":"127.0.0.1:59390_solr",
          "base_url":"http://127.0.0.1:59390/solr",
          "state":"active",
          "collection":"awhollynewcollection_0",
          "shard":"shard3",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node24":{
          "core":"awhollynewcollection_0_shard3_replica_n22",
          "leader":"true",
          "core_node_name":"core_node24",
          "node_name":"127.0.0.1:59391_solr",
          "base_url":"http://127.0.0.1:59391/solr",
          "state":"active",
          "collection":"awhollynewcollection_0",
          "shard":"shard3",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node33":{
          "core":"awhollynewcollection_0_shard3_replica_n31",
          "core_node_name":"core_node33",
          "node_name":"127.0.0.1:59384_solr",
          "base_url":"http://127.0.0.1:59384/solr",
          "state":"active",
          "collection":"awhollynewcollection_0",
          "shard":"shard3",
          "type":"NRT",
          "force_set_state":"false"}}},
    "shard4":{
      "range":"19990000-4ccbffff",
      "state":"active",
      "replicas":{
        "core_node30":{
          "core":"awhollynewcollection_0_shard4_replica_n28",
          "core_node_name":"core_node30",
          "node_name":"127.0.0.1:59386_solr",
          "base_url":"http://127.0.0.1:59386/solr",
          "state":"down",
          "collection":"awhollynewcollection_0",
          "shard":"shard4",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node35":{
          "core":"awhollynewcollection_0_shard4_replica_n32",
          "core_node_name":"core_node35",
          "node_name":"127.0.0.1:59384_solr",
          "base_url":"http://127.0.0.1:59384/solr",
          "state":"down",
          "collection":"awhollynewcollection_0",
          "shard":"shard4",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node39":{
          "core":"awhollynewcollection_0_shard4_replica_n36",
          "core_node_name":"core_node39",
          "node_name":"127.0.0.1:59390_solr",
          "base_url":"http://127.0.0.1:59390/solr",
          "state":"down",
          "collection":"awhollynewcollection_0",
          "shard":"shard4",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node40":{
          "core":"awhollynewcollection_0_shard4_replica_n38",
          "leader":"true",
          "core_node_name":"core_node40",
          "node_name":"127.0.0.1:59391_solr",
          "base_url":"http://127.0.0.1:59391/solr",
          "state":"down",
          "collection":"awhollynewcollection_0",
          "shard":"shard4",
          "type":"NRT",
          "force_set_state":"false"}}},
    "shard5":{
      "range":"4ccc0000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node3":{
          "core":"awhollynewcollection_0_shard5_replica_n1",
          "leader":"true",
          "core_node_name":"core_node3",
          "node_name":"127.0.0.1:59391_solr",
          "base_url":"http://127.0.0.1:59391/solr",
          "state":"active",
          "collection":"awhollynewcollection_0",
          "shard":"shard5",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node13":{
          "core":"awhollynewcollection_0_shard5_replica_n10",
          "core_node_name":"core_node13",
          "node_name":"127.0.0.1:59384_solr",
          "base_url":"http://127.0.0.1:59384/solr",
          "state":"active",
          "collection":"awhollynewcollection_0",
          "shard":"shard5",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node23":{
          "core":"awhollynewcollection_0_shard5_replica_n20",
          "core_node_name":"core_node23",
          "node_name":"127.0.0.1:59386_solr",
          "base_url":"http://127.0.0.1:59386/solr",
          "state":"active",
          "collection":"awhollynewcollection_0",
          "shard":"shard5",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node27":{
          "core":"awhollynewcollection_0_shard5_replica_n25",
          "core_node_name":"core_node27",
          "node_name":"127.0.0.1:59390_solr",
          "base_url":"http://127.0.0.1:59390/solr",
          "state":"active",
          "collection":"awhollynewcollection_0",
          "shard":"shard5",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"4",
  "tlogReplicas":"0"}

Stack Trace:
java.lang.RuntimeException: Failed while waiting for active collection
Timeout waiting to see state for collection=awhollynewcollection_0 :DocCollection(awhollynewcollection_0//collections/awhollynewcollection_0/state.json/20 )={
  "pullReplicas":"0",
  "configName":"conf2",
  "replicationFactor":"4",
  "shards":{
    "shard1":{
      "range":"80000000-b332ffff",
      "state":"active",
      "replicas":{
        "core_node11":{
          "core":"awhollynewcollection_0_shard1_replica_n8",
          "leader":"true",
          "core_node_name":"core_node11",
          "node_name":"127.0.0.1:59391_solr",
          "base_url":"http://127.0.0.1:59391/solr",
          "state":"active",
          "collection":"awhollynewcollection_0",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node19":{
          "core":"awhollynewcollection_0_shard1_replica_n16",
          "core_node_name":"core_node19",
          "node_name":"127.0.0.1:59384_solr",
          "base_url":"http://127.0.0.1:59384/solr",
          "state":"active",
          "collection":"awhollynewcollection_0",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node21":{
          "core":"awhollynewcollection_0_shard1_replica_n18",
          "core_node_name":"core_node21",
          "node_name":"127.0.0.1:59390_solr",
          "base_url":"http://127.0.0.1:59390/solr",
          "state":"active",
          "collection":"awhollynewcollection_0",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node37":{
          "core":"awhollynewcollection_0_shard1_replica_n34",
          "core_node_name":"core_node37",
          "node_name":"127.0.0.1:59386_solr",
          "base_url":"http://127.0.0.1:59386/solr",
          "state":"active",
          "collection":"awhollynewcollection_0",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"}}},
    "shard2":{
      "range":"b3330000-e665ffff",
      "state":"active",
      "replicas":{
        "core_node7":{
          "core":"awhollynewcollection_0_shard2_replica_n4",
          "core_node_name":"core_node7",
          "node_name":"127.0.0.1:59384_solr",
          "base_url":"http://127.0.0.1:59384/solr",
          "state":"active",
          "collection":"awhollynewcollection_0",
          "shard":"shard2",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node9":{
          "core":"awhollynewcollection_0_shard2_replica_n6",
          "core_node_name":"core_node9",
          "node_name":"127.0.0.1:59386_solr",
          "base_url":"http://127.0.0.1:59386/solr",
          "state":"active",
          "collection":"awhollynewcollection_0",
          "shard":"shard2",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node15":{
          "core":"awhollynewcollection_0_shard2_replica_n12",
          "core_node_name":"core_node15",
          "node_name":"127.0.0.1:59390_solr",
          "base_url":"http://127.0.0.1:59390/solr",
          "state":"active",
          "collection":"awhollynewcollection_0",
          "shard":"shard2",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node29":{
          "core":"awhollynewcollection_0_shard2_replica_n26",
          "leader":"true",
          "core_node_name":"core_node29",
          "node_name":"127.0.0.1:59391_solr",
          "base_url":"http://127.0.0.1:59391/solr",
          "state":"active",
          "collection":"awhollynewcollection_0",
          "shard":"shard2",
          "type":"NRT",
          "force_set_state":"false"}}},
    "shard3":{
      "range":"e6660000-1998ffff",
      "state":"active",
      "replicas":{
        "core_node5":{
          "core":"awhollynewcollection_0_shard3_replica_n2",
          "core_node_name":"core_node5",
          "node_name":"127.0.0.1:59386_solr",
          "base_url":"http://127.0.0.1:59386/solr",
          "state":"active",
          "collection":"awhollynewcollection_0",
          "shard":"shard3",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node17":{
          "core":"awhollynewcollection_0_shard3_replica_n14",
          "core_node_name":"core_node17",
          "node_name":"127.0.0.1:59390_solr",
          "base_url":"http://127.0.0.1:59390/solr",
          "state":"active",
          "collection":"awhollynewcollection_0",
          "shard":"shard3",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node24":{
          "core":"awhollynewcollection_0_shard3_replica_n22",
          "leader":"true",
          "core_node_name":"core_node24",
          "node_name":"127.0.0.1:59391_solr",
          "base_url":"http://127.0.0.1:59391/solr",
          "state":"active",
          "collection":"awhollynewcollection_0",
          "shard":"shard3",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node33":{
          "core":"awhollynewcollection_0_shard3_replica_n31",
          "core_node_name":"core_node33",
          "node_name":"127.0.0.1:59384_solr",
          "base_url":"http://127.0.0.1:59384/solr",
          "state":"active",
          "collection":"awhollynewcollection_0",
          "shard":"shard3",
          "type":"NRT",
          "force_set_state":"false"}}},
    "shard4":{
      "range":"19990000-4ccbffff",
      "state":"active",
      "replicas":{
        "core_node30":{
          "core":"awhollynewcollection_0_shard4_replica_n28",
          "core_node_name":"core_node30",
          "node_name":"127.0.0.1:59386_solr",
          "base_url":"http://127.0.0.1:59386/solr",
          "state":"down",
          "collection":"awhollynewcollection_0",
          "shard":"shard4",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node35":{
          "core":"awhollynewcollection_0_shard4_replica_n32",
          "core_node_name":"core_node35",
          "node_name":"127.0.0.1:59384_solr",
          "base_url":"http://127.0.0.1:59384/solr",
          "state":"down",
          "collection":"awhollynewcollection_0",
          "shard":"shard4",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node39":{
          "core":"awhollynewcollection_0_shard4_replica_n36",
          "core_node_name":"core_node39",
          "node_name":"127.0.0.1:59390_solr",
          "base_url":"http://127.0.0.1:59390/solr",
          "state":"down",
          "collection":"awhollynewcollection_0",
          "shard":"shard4",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node40":{
          "core":"awhollynewcollection_0_shard4_replica_n38",
          "leader":"true",
          "core_node_name":"core_node40",
          "node_name":"127.0.0.1:59391_solr",
          "base_url":"http://127.0.0.1:59391/solr",
          "state":"down",
          "collection":"awhollynewcollection_0",
          "shard":"shard4",
          "type":"NRT",
          "force_set_state":"false"}}},
    "shard5":{
      "range":"4ccc0000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node3":{
          "core":"awhollynewcollection_0_shard5_replica_n1",
          "leader":"true",
          "core_node_name":"core_node3",
          "node_name":"127.0.0.1:59391_solr",
          "base_url":"http://127.0.0.1:59391/solr",
          "state":"active",
          "collection":"awhollynewcollection_0",
          "shard":"shard5",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node13":{
          "core":"awhollynewcollection_0_shard5_replica_n10",
          "core_node_name":"core_node13",
          "node_name":"127.0.0.1:59384_solr",
          "base_url":"http://127.0.0.1:59384/solr",
          "state":"active",
          "collection":"awhollynewcollection_0",
          "shard":"shard5",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node23":{
          "core":"awhollynewcollection_0_shard5_replica_n20",
          "core_node_name":"core_node23",
          "node_name":"127.0.0.1:59386_solr",
          "base_url":"http://127.0.0.1:59386/solr",
          "state":"active",
          "collection":"awhollynewcollection_0",
          "shard":"shard5",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node27":{
          "core":"awhollynewcollection_0_shard5_replica_n25",
          "core_node_name":"core_node27",
          "node_name":"127.0.0.1:59390_solr",
          "base_url":"http://127.0.0.1:59390/solr",
          "state":"active",
          "collection":"awhollynewcollection_0",
          "shard":"shard5",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"4",
  "tlogReplicas":"0"}
Live Nodes: [127.0.0.1:59384_solr, 127.0.0.1:59386_solr, 127.0.0.1:59390_solr, 127.0.0.1:59391_solr]
Last available state: DocCollection(awhollynewcollection_0//collections/awhollynewcollection_0/state.json/20 )={
  "pullReplicas":"0",
  "configName":"conf2",
  "replicationFactor":"4",
  "shards":{
    "shard1":{
      "range":"80000000-b332ffff",
      "state":"active",
      "replicas":{
        "core_node11":{
          "core":"awhollynewcollection_0_shard1_replica_n8",
          "leader":"true",
          "core_node_name":"core_node11",
          "node_name":"127.0.0.1:59391_solr",
          "base_url":"http://127.0.0.1:59391/solr",
          "state":"active",
          "collection":"awhollynewcollection_0",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node19":{
          "core":"awhollynewcollection_0_shard1_replica_n16",
          "core_node_name":"core_node19",
          "node_name":"127.0.0.1:59384_solr",
          "base_url":"http://127.0.0.1:59384/solr",
          "state":"active",
          "collection":"awhollynewcollection_0",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node21":{
          "core":"awhollynewcollection_0_shard1_replica_n18",
          "core_node_name":"core_node21",
          "node_name":"127.0.0.1:59390_solr",
          "base_url":"http://127.0.0.1:59390/solr",
          "state":"active",
          "collection":"awhollynewcollection_0",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node37":{
          "core":"awhollynewcollection_0_shard1_replica_n34",
          "core_node_name":"core_node37",
          "node_name":"127.0.0.1:59386_solr",
          "base_url":"http://127.0.0.1:59386/solr",
          "state":"active",
          "collection":"awhollynewcollection_0",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"}}},
    "shard2":{
      "range":"b3330000-e665ffff",
      "state":"active",
      "replicas":{
        "core_node7":{
          "core":"awhollynewcollection_0_shard2_replica_n4",
          "core_node_name":"core_node7",
          "node_name":"127.0.0.1:59384_solr",
          "base_url":"http://127.0.0.1:59384/solr",
          "state":"active",
          "collection":"awhollynewcollection_0",
          "shard":"shard2",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node9":{
          "core":"awhollynewcollection_0_shard2_replica_n6",
          "core_node_name":"core_node9",
          "node_name":"127.0.0.1:59386_solr",
          "base_url":"http://127.0.0.1:59386/solr",
          "state":"active",
          "collection":"awhollynewcollection_0",
          "shard":"shard2",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node15":{
          "core":"awhollynewcollection_0_shard2_replica_n12",
          "core_node_name":"core_node15",
          "node_name":"127.0.0.1:59390_solr",
          "base_url":"http://127.0.0.1:59390/solr",
          "state":"active",
          "collection":"awhollynewcollection_0",
          "shard":"shard2",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node29":{
          "core":"awhollynewcollection_0_shard2_replica_n26",
          "leader":"true",
          "core_node_name":"core_node29",
          "node_name":"127.0.0.1:59391_solr",
          "base_url":"http://127.0.0.1:59391/solr",
          "state":"active",
          "collection":"awhollynewcollection_0",
          "shard":"shard2",
          "type":"NRT",
          "force_set_state":"false"}}},
    "shard3":{
      "range":"e6660000-1998ffff",
      "state":"active",
      "replicas":{
        "core_node5":{
          "core":"awhollynewcollection_0_shard3_replica_n2",
          "core_node_name":"core_node5",
          "node_name":"127.0.0.1:59386_solr",
          "base_url":"http://127.0.0.1:59386/solr",
          "state":"active",
          "collection":"awhollynewcollection_0",
          "shard":"shard3",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node17":{
          "core":"awhollynewcollection_0_shard3_replica_n14",
          "core_node_name":"core_node17",
          "node_name":"127.0.0.1:59390_solr",
          "base_url":"http://127.0.0.1:59390/solr",
          "state":"active",
          "collection":"awhollynewcollection_0",
          "shard":"shard3",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node24":{
          "core":"awhollynewcollection_0_shard3_replica_n22",
          "leader":"true",
          "core_node_name":"core_node24",
          "node_name":"127.0.0.1:59391_solr",
          "base_url":"http://127.0.0.1:59391/solr",
          "state":"active",
          "collection":"awhollynewcollection_0",
          "shard":"shard3",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node33":{
          "core":"awhollynewcollection_0_shard3_replica_n31",
          "core_node_name":"core_node33",
          "node_name":"127.0.0.1:59384_solr",
          "base_url":"http://127.0.0.1:59384/solr",
          "state":"active",
          "collection":"awhollynewcollection_0",
          "shard":"shard3",
          "type":"NRT",
          "force_set_state":"false"}}},
    "shard4":{
      "range":"19990000-4ccbffff",
      "state":"active",
      "replicas":{
        "core_node30":{
          "core":"awhollynewcollection_0_shard4_replica_n28",
          "core_node_name":"core_node30",
          "node_name":"127.0.0.1:59386_solr",
          "base_url":"http://127.0.0.1:59386/solr",
          "state":"down",
          "collection":"awhollynewcollection_0",
          "shard":"shard4",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node35":{
          "core":"awhollynewcollection_0_shard4_replica_n32",
          "core_node_name":"core_node35",
          "node_name":"127.0.0.1:59384_solr",
          "base_url":"http://127.0.0.1:59384/solr",
          "state":"down",
          "collection":"awhollynewcollection_0",
          "shard":"shard4",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node39":{
          "core":"awhollynewcollection_0_shard4_replica_n36",
          "core_node_name":"core_node39",
          "node_name":"127.0.0.1:59390_solr",
          "base_url":"http://127.0.0.1:59390/solr",
          "state":"down",
          "collection":"awhollynewcollection_0",
          "shard":"shard4",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node40":{
          "core":"awhollynewcollection_0_shard4_replica_n38",
          "leader":"true",
          "core_node_name":"core_node40",
          "node_name":"127.0.0.1:59391_solr",
          "base_url":"http://127.0.0.1:59391/solr",
          "state":"down",
          "collection":"awhollynewcollection_0",
          "shard":"shard4",
          "type":"NRT",
          "force_set_state":"false"}}},
    "shard5":{
      "range":"4ccc0000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node3":{
          "core":"awhollynewcollection_0_shard5_replica_n1",
          "leader":"true",
          "core_node_name":"core_node3",
          "node_name":"127.0.0.1:59391_solr",
          "base_url":"http://127.0.0.1:59391/solr",
          "state":"active",
          "collection":"awhollynewcollection_0",
          "shard":"shard5",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node13":{
          "core":"awhollynewcollection_0_shard5_replica_n10",
          "core_node_name":"core_node13",
          "node_name":"127.0.0.1:59384_solr",
          "base_url":"http://127.0.0.1:59384/solr",
          "state":"active",
          "collection":"awhollynewcollection_0",
          "shard":"shard5",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node23":{
          "core":"awhollynewcollection_0_shard5_replica_n20",
          "core_node_name":"core_node23",
          "node_name":"127.0.0.1:59386_solr",
          "base_url":"http://127.0.0.1:59386/solr",
          "state":"active",
          "collection":"awhollynewcollection_0",
          "shard":"shard5",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node27":{
          "core":"awhollynewcollection_0_shard5_replica_n25",
          "core_node_name":"core_node27",
          "node_name":"127.0.0.1:59390_solr",
          "base_url":"http://127.0.0.1:59390/solr",
          "state":"active",
          "collection":"awhollynewcollection_0",
          "shard":"shard5",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"4",
  "tlogReplicas":"0"}
	at __randomizedtesting.SeedInfo.seed([58C05331E518725E:10B52785E32B5DCB]:0)
	at org.apache.solr.cloud.MiniSolrCloudCluster.waitForActiveCollection(MiniSolrCloudCluster.java:880)
	at org.apache.solr.cloud.MiniSolrCloudCluster.waitForActiveCollection(MiniSolrCloudCluster.java:885)
	at org.apache.solr.cloud.api.collections.AbstractCollectionsAPIDistributedZkTestBase.testCollectionsAPI(AbstractCollectionsAPIDistributedZkTestBase.java:439)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:78)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:567)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base/java.lang.Thread.run(Thread.java:831)


FAILED:  org.apache.solr.security.AuthWithShardHandlerFactoryOverrideTest.collectionTest

Error Message:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:50448/solr: no servers hosting shard: shard4

Stack Trace:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:50448/solr: no servers hosting shard: shard4
	at __randomizedtesting.SeedInfo.seed([58C05331E518725E:8D915513765D39A3]:0)
	at app//org.apache.solr.client.solrj.impl.Http2SolrClient.processErrorsAndResponse(Http2SolrClient.java:822)
	at app//org.apache.solr.client.solrj.impl.Http2SolrClient.processErrorsAndResponse(Http2SolrClient.java:485)
	at app//org.apache.solr.client.solrj.impl.Http2SolrClient.request(Http2SolrClient.java:453)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:223)
	at app//org.apache.solr.security.AuthWithShardHandlerFactoryOverrideTest.collectionTest(AuthWithShardHandlerFactoryOverrideTest.java:92)
	at java.base@16.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base@16.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:78)
	at java.base@16.0.2/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base@16.0.2/java.lang.reflect.Method.invoke(Method.java:567)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)

[JENKINS] Solr-main-MacOSX (64bit/jdk-16.0.2) - Build # 873 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/873/
Java: 64bit/jdk-16.0.2 -XX:+UseCompressedOops -XX:+UseParallelGC

2 tests failed.
FAILED:  org.apache.solr.handler.TestSolrConfigHandlerCloud.test

Error Message:
java.lang.AssertionError: {
  "responseHeader":{
    "status":500,
    "QTime":30258},
  "errorMessages":["2 out of 5 the property overlay to be of version 20 within 30 seconds! Failed cores: [http://127.0.0.1:63786/whyyu/t/collection1_shard1_replica_n1/, http://127.0.0.1:63816/whyyu/t/collection1_shard1_replica_n6/]\n"],
  "WARNING":"This response format is experimental.  It is likely to change in the future.",
  "error":{
    "metadata":[
      "error-class","org.apache.solr.common.SolrException",
      "root-error-class","org.apache.solr.common.SolrException"],
    "msg":"2 out of 5 the property overlay to be of version 20 within 30 seconds! Failed cores: [http://127.0.0.1:63786/whyyu/t/collection1_shard1_replica_n1/, http://127.0.0.1:63816/whyyu/t/collection1_shard1_replica_n6/]",
    "trace":"org.apache.solr.common.SolrException: 2 out of 5 the property overlay to be of version 20 within 30 seconds! Failed cores: [http://127.0.0.1:63786/whyyu/t/collection1_shard1_replica_n1/, http://127.0.0.1:63816/whyyu/t/collection1_shard1_replica_n6/]\n\tat org.apache.solr.handler.SolrConfigHandler.waitForAllReplicasState(SolrConfigHandler.java:892)\n\tat org.apache.solr.handler.SolrConfigHandler$Command.handleCommands(SolrConfigHandler.java:591)\n\tat org.apache.solr.handler.SolrConfigHandler$Command.handlePOST(SolrConfigHandler.java:404)\n\tat org.apache.solr.handler.SolrConfigHandler.handleRequestBody(SolrConfigHandler.java:143)\n\tat org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:207)\n\tat org.apache.solr.core.SolrCore.execute(SolrCore.java:2866)\n\tat org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:887)\n\tat org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:606)\n\tat org.apache.solr.servlet.SolrDispatchFilter.dispatch(SolrDispatchFilter.java:239)\n\tat org.apache.solr.servlet.SolrDispatchFilter.lambda$doFilter$0(SolrDispatchFilter.java:207)\n\tat org.apache.solr.servlet.ServletUtils.traceHttpRequestExecution2(ServletUtils.java:257)\n\tat org.apache.solr.servlet.ServletUtils.rateLimitRequest(ServletUtils.java:227)\n\tat org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:202)\n\tat org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:184)\n\tat org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)\n\tat org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)\n\tat org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:187)\n\tat org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)\n\tat org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)\n\tat org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)\n\tat org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)\n\tat org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1434)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)\n\tat org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)\n\tat org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)\n\tat org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1349)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)\n\tat org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)\n\tat org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)\n\tat org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:713)\n\tat org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)\n\tat org.eclipse.jetty.server.Server.handle(Server.java:516)\n\tat org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:400)\n\tat org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:645)\n\tat org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:392)\n\tat org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:277)\n\tat org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311)\n\tat org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:105)\n\tat org.eclipse.jetty.io.ChannelEndPoint$1.run(ChannelEndPoint.java:104)\n\tat org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)\n\tat org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)\n\tat java.base/java.lang.Thread.run(Thread.java:831)\n",
    "code":500}}
 expected null, but was:<[2 out of 5 the property overlay to be of version 20 within 30 seconds! Failed cores: [http://127.0.0.1:63786/whyyu/t/collection1_shard1_replica_n1/, http://127.0.0.1:63816/whyyu/t/collection1_shard1_replica_n6/]
]>

Stack Trace:
java.lang.AssertionError: {
  "responseHeader":{
    "status":500,
    "QTime":30258},
  "errorMessages":["2 out of 5 the property overlay to be of version 20 within 30 seconds! Failed cores: [http://127.0.0.1:63786/whyyu/t/collection1_shard1_replica_n1/, http://127.0.0.1:63816/whyyu/t/collection1_shard1_replica_n6/]\n"],
  "WARNING":"This response format is experimental.  It is likely to change in the future.",
  "error":{
    "metadata":[
      "error-class","org.apache.solr.common.SolrException",
      "root-error-class","org.apache.solr.common.SolrException"],
    "msg":"2 out of 5 the property overlay to be of version 20 within 30 seconds! Failed cores: [http://127.0.0.1:63786/whyyu/t/collection1_shard1_replica_n1/, http://127.0.0.1:63816/whyyu/t/collection1_shard1_replica_n6/]",
    "trace":"org.apache.solr.common.SolrException: 2 out of 5 the property overlay to be of version 20 within 30 seconds! Failed cores: [http://127.0.0.1:63786/whyyu/t/collection1_shard1_replica_n1/, http://127.0.0.1:63816/whyyu/t/collection1_shard1_replica_n6/]\n\tat org.apache.solr.handler.SolrConfigHandler.waitForAllReplicasState(SolrConfigHandler.java:892)\n\tat org.apache.solr.handler.SolrConfigHandler$Command.handleCommands(SolrConfigHandler.java:591)\n\tat org.apache.solr.handler.SolrConfigHandler$Command.handlePOST(SolrConfigHandler.java:404)\n\tat org.apache.solr.handler.SolrConfigHandler.handleRequestBody(SolrConfigHandler.java:143)\n\tat org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:207)\n\tat org.apache.solr.core.SolrCore.execute(SolrCore.java:2866)\n\tat org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:887)\n\tat org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:606)\n\tat org.apache.solr.servlet.SolrDispatchFilter.dispatch(SolrDispatchFilter.java:239)\n\tat org.apache.solr.servlet.SolrDispatchFilter.lambda$doFilter$0(SolrDispatchFilter.java:207)\n\tat org.apache.solr.servlet.ServletUtils.traceHttpRequestExecution2(ServletUtils.java:257)\n\tat org.apache.solr.servlet.ServletUtils.rateLimitRequest(ServletUtils.java:227)\n\tat org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:202)\n\tat org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:184)\n\tat org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)\n\tat org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)\n\tat org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:187)\n\tat org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)\n\tat org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)\n\tat org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)\n\tat org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)\n\tat org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1434)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)\n\tat org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)\n\tat org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)\n\tat org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1349)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)\n\tat org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)\n\tat org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)\n\tat org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:713)\n\tat org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)\n\tat org.eclipse.jetty.server.Server.handle(Server.java:516)\n\tat org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:400)\n\tat org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:645)\n\tat org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:392)\n\tat org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:277)\n\tat org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311)\n\tat org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:105)\n\tat org.eclipse.jetty.io.ChannelEndPoint$1.run(ChannelEndPoint.java:104)\n\tat org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)\n\tat org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)\n\tat java.base/java.lang.Thread.run(Thread.java:831)\n",
    "code":500}}
 expected null, but was:<[2 out of 5 the property overlay to be of version 20 within 30 seconds! Failed cores: [http://127.0.0.1:63786/whyyu/t/collection1_shard1_replica_n1/, http://127.0.0.1:63816/whyyu/t/collection1_shard1_replica_n6/]
]>
	at __randomizedtesting.SeedInfo.seed([67A7A4422BA745D9:EFF39B98855B2821]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.junit.Assert.failNotNull(Assert.java:756)
	at org.junit.Assert.assertNull(Assert.java:738)
	at org.apache.solr.core.TestSolrConfigHandler.runConfigCommand(TestSolrConfigHandler.java:226)
	at org.apache.solr.handler.TestSolrConfigHandlerCloud.testReqParams(TestSolrConfigHandlerCloud.java:130)
	at org.apache.solr.handler.TestSolrConfigHandlerCloud.test(TestSolrConfigHandlerCloud.java:51)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:78)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:567)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1164)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1135)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base/java.lang.Thread.run(Thread.java:831)


FAILED:  org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=1179, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:361)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:75)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getLines(AbstractAnalysisFactory.java:302)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getWordSet(AbstractAnalysisFactory.java:293)
        at app//org.apache.lucene.analysis.util.ElisionFilterFactory.inform(ElisionFilterFactory.java:73)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:762)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1470)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1319)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1307)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:654)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:188)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:119)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:279)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:345)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:256)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$629/0x000000080114e378.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:49)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:252)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1850)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3394)
        at app//org.apache.solr.core.SolrCore$$Lambda$740/0x0000000801211c98.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2761)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1195/0x000000080164a920.run(Unknown Source)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=1179, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:361)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:75)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getLines(AbstractAnalysisFactory.java:302)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getWordSet(AbstractAnalysisFactory.java:293)
        at app//org.apache.lucene.analysis.util.ElisionFilterFactory.inform(ElisionFilterFactory.java:73)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:762)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1470)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1319)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1307)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:654)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:188)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:119)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:279)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:345)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:256)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$629/0x000000080114e378.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:49)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:252)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1850)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3394)
        at app//org.apache.solr.core.SolrCore$$Lambda$740/0x0000000801211c98.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2761)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1195/0x000000080164a920.run(Unknown Source)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
	at __randomizedtesting.SeedInfo.seed([67A7A4422BA745D9]:0)

[JENKINS] Solr-main-MacOSX (64bit/jdk-17.0.2) - Build # 872 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/872/
Java: 64bit/jdk-17.0.2 -XX:+UseCompressedOops -XX:+UseSerialGC

10 tests failed.
FAILED:  org.apache.solr.schema.ManagedSchemaRoundRobinCloudTest.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.schema.ManagedSchemaRoundRobinCloudTest: 
   1) Thread[id=11667, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-ManagedSchemaRoundRobinCloudTest]
        at java.base@17.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:318)
        at app//org.apache.solr.core.RequestParams.getFreshRequestParams(RequestParams.java:152)
        at app//org.apache.solr.core.SolrConfig.refreshRequestParams(SolrConfig.java:1145)
        at app//org.apache.solr.core.SolrConfig.getRequestParams(SolrConfig.java:1123)
        at app//org.apache.solr.core.SolrConfig.<init>(SolrConfig.java:254)
        at app//org.apache.solr.core.SolrConfig.readFromResourceLoader(SolrConfig.java:158)
        at app//org.apache.solr.core.ConfigSetService.createSolrConfig(ConfigSetService.java:297)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:250)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1850)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3394)
        at app//org.apache.solr.core.SolrCore$$Lambda$1042/0x000000080152cb98.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2761)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1824/0x0000000801864b10.run(Unknown Source)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.schema.ManagedSchemaRoundRobinCloudTest: 
   1) Thread[id=11667, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-ManagedSchemaRoundRobinCloudTest]
        at java.base@17.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:318)
        at app//org.apache.solr.core.RequestParams.getFreshRequestParams(RequestParams.java:152)
        at app//org.apache.solr.core.SolrConfig.refreshRequestParams(SolrConfig.java:1145)
        at app//org.apache.solr.core.SolrConfig.getRequestParams(SolrConfig.java:1123)
        at app//org.apache.solr.core.SolrConfig.<init>(SolrConfig.java:254)
        at app//org.apache.solr.core.SolrConfig.readFromResourceLoader(SolrConfig.java:158)
        at app//org.apache.solr.core.ConfigSetService.createSolrConfig(ConfigSetService.java:297)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:250)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1850)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3394)
        at app//org.apache.solr.core.SolrCore$$Lambda$1042/0x000000080152cb98.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2761)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1824/0x0000000801864b10.run(Unknown Source)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
	at __randomizedtesting.SeedInfo.seed([86B33B7A150BDCB0]:0)


FAILED:  org.apache.solr.schema.SchemaApiFailureTest.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=12121, name=qtp172511196-12121, state=RUNNABLE, group=TGRP-SchemaApiFailureTest], registration stack trace below.

Stack Trace:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=12121, name=qtp172511196-12121, state=RUNNABLE, group=TGRP-SchemaApiFailureTest], registration stack trace below.
	at java.base@17.0.2/java.lang.Thread.getStackTrace(Thread.java:1610)
	at app//com.carrotsearch.randomizedtesting.RandomizedContext.closeAtEnd(RandomizedContext.java:157)
	at app//org.apache.lucene.util.LuceneTestCase.closeAfterSuite(LuceneTestCase.java:798)
	at app//org.apache.lucene.util.LuceneTestCase.wrapDirectory(LuceneTestCase.java:1504)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1368)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1350)
	at app//org.apache.solr.core.MockDirectoryFactory.create(MockDirectoryFactory.java:51)
	at app//org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:368)
	at app//org.apache.solr.core.SolrCore.initSnapshotMetaDataManager(SolrCore.java:606)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1119)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1048)
	at app//org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1562)
	at app//org.apache.solr.core.CoreContainer.create(CoreContainer.java:1428)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:389)
	at app//org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:422)
	at app//org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:205)
	at app//org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:207)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:941)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:893)
	at app//org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:584)
	at app//org.apache.solr.servlet.SolrDispatchFilter.dispatch(SolrDispatchFilter.java:239)
	at app//org.apache.solr.servlet.SolrDispatchFilter.lambda$doFilter$0(SolrDispatchFilter.java:207)
	at app//org.apache.solr.servlet.ServletUtils.traceHttpRequestExecution2(ServletUtils.java:257)
	at app//org.apache.solr.servlet.ServletUtils.rateLimitRequest(ServletUtils.java:227)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:202)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:184)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:187)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1434)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
	at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)
	at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1349)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
	at app//org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:713)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.server.Server.handle(Server.java:516)
	at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:400)
	at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:645)
	at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:392)
	at app//org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:352)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
	at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
Caused by: java.lang.AssertionError: Directory not closed: MockDirectoryWrapper(ByteBuffersDirectory@3e5f5862 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@77190caa)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.lucene.util.CloseableDirectory.close(CloseableDirectory.java:43)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:789)
	at com.carrotsearch.randomizedtesting.RandomizedContext.closeResources(RandomizedContext.java:225)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.afterAlways(RandomizedRunner.java:805)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	... 1 more


FAILED:  org.apache.solr.schema.SchemaApiFailureTest.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=12121, name=qtp172511196-12121, state=RUNNABLE, group=TGRP-SchemaApiFailureTest], registration stack trace below.

Stack Trace:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=12121, name=qtp172511196-12121, state=RUNNABLE, group=TGRP-SchemaApiFailureTest], registration stack trace below.
	at java.base@17.0.2/java.lang.Thread.getStackTrace(Thread.java:1610)
	at app//com.carrotsearch.randomizedtesting.RandomizedContext.closeAtEnd(RandomizedContext.java:157)
	at app//org.apache.lucene.util.LuceneTestCase.closeAfterSuite(LuceneTestCase.java:798)
	at app//org.apache.lucene.util.LuceneTestCase.wrapDirectory(LuceneTestCase.java:1504)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1368)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1350)
	at app//org.apache.solr.core.MockDirectoryFactory.create(MockDirectoryFactory.java:51)
	at app//org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:368)
	at app//org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:412)
	at app//org.apache.solr.core.SolrCore.initIndex(SolrCore.java:856)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1123)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1048)
	at app//org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1562)
	at app//org.apache.solr.core.CoreContainer.create(CoreContainer.java:1428)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:389)
	at app//org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:422)
	at app//org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:205)
	at app//org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:207)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:941)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:893)
	at app//org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:584)
	at app//org.apache.solr.servlet.SolrDispatchFilter.dispatch(SolrDispatchFilter.java:239)
	at app//org.apache.solr.servlet.SolrDispatchFilter.lambda$doFilter$0(SolrDispatchFilter.java:207)
	at app//org.apache.solr.servlet.ServletUtils.traceHttpRequestExecution2(ServletUtils.java:257)
	at app//org.apache.solr.servlet.ServletUtils.rateLimitRequest(ServletUtils.java:227)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:202)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:184)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:187)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1434)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
	at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)
	at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1349)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
	at app//org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:713)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.server.Server.handle(Server.java:516)
	at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:400)
	at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:645)
	at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:392)
	at app//org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:352)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
	at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
Caused by: java.lang.AssertionError: Directory not closed: MockDirectoryWrapper(ByteBuffersDirectory@19f969f0 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2f7c8ba5)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.lucene.util.CloseableDirectory.close(CloseableDirectory.java:43)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:789)
	at com.carrotsearch.randomizedtesting.RandomizedContext.closeResources(RandomizedContext.java:225)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.afterAlways(RandomizedRunner.java:805)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	... 1 more


FAILED:  org.apache.solr.schema.SchemaApiFailureTest.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=12121, name=qtp172511196-12121, state=RUNNABLE, group=TGRP-SchemaApiFailureTest], registration stack trace below.

Stack Trace:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=12121, name=qtp172511196-12121, state=RUNNABLE, group=TGRP-SchemaApiFailureTest], registration stack trace below.
	at java.base@17.0.2/java.lang.Thread.getStackTrace(Thread.java:1610)
	at app//com.carrotsearch.randomizedtesting.RandomizedContext.closeAtEnd(RandomizedContext.java:157)
	at app//org.apache.lucene.util.LuceneTestCase.closeAfterSuite(LuceneTestCase.java:798)
	at app//org.apache.lucene.util.LuceneTestCase.wrapDirectory(LuceneTestCase.java:1504)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1368)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1350)
	at app//org.apache.solr.core.MockDirectoryFactory.create(MockDirectoryFactory.java:51)
	at app//org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:368)
	at app//org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:108)
	at app//org.apache.solr.core.SolrCore.initIndex(SolrCore.java:895)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1123)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1048)
	at app//org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1562)
	at app//org.apache.solr.core.CoreContainer.create(CoreContainer.java:1428)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:389)
	at app//org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:422)
	at app//org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:205)
	at app//org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:207)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:941)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:893)
	at app//org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:584)
	at app//org.apache.solr.servlet.SolrDispatchFilter.dispatch(SolrDispatchFilter.java:239)
	at app//org.apache.solr.servlet.SolrDispatchFilter.lambda$doFilter$0(SolrDispatchFilter.java:207)
	at app//org.apache.solr.servlet.ServletUtils.traceHttpRequestExecution2(ServletUtils.java:257)
	at app//org.apache.solr.servlet.ServletUtils.rateLimitRequest(ServletUtils.java:227)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:202)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:184)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:187)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1434)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
	at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)
	at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1349)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
	at app//org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:713)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.server.Server.handle(Server.java:516)
	at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:400)
	at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:645)
	at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:392)
	at app//org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:352)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
	at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
Caused by: java.lang.AssertionError: Directory not closed: MockDirectoryWrapper(ByteBuffersDirectory@8a516e lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7a48bb7e)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.lucene.util.CloseableDirectory.close(CloseableDirectory.java:43)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:789)
	at com.carrotsearch.randomizedtesting.RandomizedContext.closeResources(RandomizedContext.java:225)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.afterAlways(RandomizedRunner.java:805)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	... 1 more


FAILED:  org.apache.solr.schema.SchemaApiFailureTest.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.schema.SchemaApiFailureTest: 
   1) Thread[id=12157, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-SchemaApiFailureTest]
        at java.base@17.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:328)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.lookupZKManagedSchemaPath(ManagedIndexSchemaFactory.java:125)
        at app//org.apache.solr.schema.ZkIndexSchemaReader.<init>(ZkIndexSchemaReader.java:56)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.inform(ManagedIndexSchemaFactory.java:537)
        at app//org.apache.solr.core.SolrResourceLoader.inform(SolrResourceLoader.java:728)
        at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1155)
        at app//org.apache.solr.core.SolrCore.reload(SolrCore.java:780)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1857)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3394)
        at app//org.apache.solr.core.SolrCore$$Lambda$852/0x000000080130c378.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2761)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1269/0x00000008016b5ba0.run(Unknown Source)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   2) Thread[id=12172, name=searcherExecutor-8725-thread-1-processing-127.0.0.1:58021_solr, state=WAITING, group=TGRP-SchemaApiFailureTest]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
        at java.base@17.0.2/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:230)
        at app//org.apache.solr.core.SolrCore.lambda$new$2(SolrCore.java:1142)
        at app//org.apache.solr.core.SolrCore$$Lambda$474/0x00000008010cc810.call(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.FutureTask.run(FutureTask.java:264)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$268/0x0000000800f84108.run(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.schema.SchemaApiFailureTest: 
   1) Thread[id=12157, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-SchemaApiFailureTest]
        at java.base@17.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:328)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.lookupZKManagedSchemaPath(ManagedIndexSchemaFactory.java:125)
        at app//org.apache.solr.schema.ZkIndexSchemaReader.<init>(ZkIndexSchemaReader.java:56)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.inform(ManagedIndexSchemaFactory.java:537)
        at app//org.apache.solr.core.SolrResourceLoader.inform(SolrResourceLoader.java:728)
        at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1155)
        at app//org.apache.solr.core.SolrCore.reload(SolrCore.java:780)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1857)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3394)
        at app//org.apache.solr.core.SolrCore$$Lambda$852/0x000000080130c378.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2761)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1269/0x00000008016b5ba0.run(Unknown Source)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   2) Thread[id=12172, name=searcherExecutor-8725-thread-1-processing-127.0.0.1:58021_solr, state=WAITING, group=TGRP-SchemaApiFailureTest]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
        at java.base@17.0.2/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:230)
        at app//org.apache.solr.core.SolrCore.lambda$new$2(SolrCore.java:1142)
        at app//org.apache.solr.core.SolrCore$$Lambda$474/0x00000008010cc810.call(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.FutureTask.run(FutureTask.java:264)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$268/0x0000000800f84108.run(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
	at __randomizedtesting.SeedInfo.seed([86B33B7A150BDCB0]:0)


FAILED:  org.apache.solr.schema.TestManagedSchemaAPI.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=4078, name=qtp537313834-4078, state=RUNNABLE, group=TGRP-TestManagedSchemaAPI], registration stack trace below.

Stack Trace:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=4078, name=qtp537313834-4078, state=RUNNABLE, group=TGRP-TestManagedSchemaAPI], registration stack trace below.
	at java.base@17.0.2/java.lang.Thread.getStackTrace(Thread.java:1610)
	at app//com.carrotsearch.randomizedtesting.RandomizedContext.closeAtEnd(RandomizedContext.java:157)
	at app//org.apache.lucene.util.LuceneTestCase.closeAfterSuite(LuceneTestCase.java:798)
	at app//org.apache.lucene.util.LuceneTestCase.wrapDirectory(LuceneTestCase.java:1504)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1368)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1350)
	at app//org.apache.solr.core.MockDirectoryFactory.create(MockDirectoryFactory.java:51)
	at app//org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:368)
	at app//org.apache.solr.core.SolrCore.initSnapshotMetaDataManager(SolrCore.java:606)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1119)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1048)
	at app//org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1562)
	at app//org.apache.solr.core.CoreContainer.create(CoreContainer.java:1428)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:389)
	at app//org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:422)
	at app//org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:205)
	at app//org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:207)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:941)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:893)
	at app//org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:584)
	at app//org.apache.solr.servlet.SolrDispatchFilter.dispatch(SolrDispatchFilter.java:239)
	at app//org.apache.solr.servlet.SolrDispatchFilter.lambda$doFilter$0(SolrDispatchFilter.java:207)
	at app//org.apache.solr.servlet.ServletUtils.traceHttpRequestExecution2(ServletUtils.java:257)
	at app//org.apache.solr.servlet.ServletUtils.rateLimitRequest(ServletUtils.java:227)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:202)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:184)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:187)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1434)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
	at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)
	at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1349)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
	at app//org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:713)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.server.Server.handle(Server.java:516)
	at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:400)
	at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:645)
	at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:392)
	at app//org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:352)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
	at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
Caused by: java.lang.AssertionError: Directory not closed: MockDirectoryWrapper(ByteBuffersDirectory@4701045 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@68a9b727)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.lucene.util.CloseableDirectory.close(CloseableDirectory.java:43)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:789)
	at com.carrotsearch.randomizedtesting.RandomizedContext.closeResources(RandomizedContext.java:225)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.afterAlways(RandomizedRunner.java:805)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	... 1 more


FAILED:  org.apache.solr.schema.TestManagedSchemaAPI.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=4078, name=qtp537313834-4078, state=RUNNABLE, group=TGRP-TestManagedSchemaAPI], registration stack trace below.

Stack Trace:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=4078, name=qtp537313834-4078, state=RUNNABLE, group=TGRP-TestManagedSchemaAPI], registration stack trace below.
	at java.base@17.0.2/java.lang.Thread.getStackTrace(Thread.java:1610)
	at app//com.carrotsearch.randomizedtesting.RandomizedContext.closeAtEnd(RandomizedContext.java:157)
	at app//org.apache.lucene.util.LuceneTestCase.closeAfterSuite(LuceneTestCase.java:798)
	at app//org.apache.lucene.util.LuceneTestCase.wrapDirectory(LuceneTestCase.java:1504)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1368)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1350)
	at app//org.apache.solr.core.MockDirectoryFactory.create(MockDirectoryFactory.java:51)
	at app//org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:368)
	at app//org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:412)
	at app//org.apache.solr.core.SolrCore.initIndex(SolrCore.java:856)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1123)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1048)
	at app//org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1562)
	at app//org.apache.solr.core.CoreContainer.create(CoreContainer.java:1428)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:389)
	at app//org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:422)
	at app//org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:205)
	at app//org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:207)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:941)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:893)
	at app//org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:584)
	at app//org.apache.solr.servlet.SolrDispatchFilter.dispatch(SolrDispatchFilter.java:239)
	at app//org.apache.solr.servlet.SolrDispatchFilter.lambda$doFilter$0(SolrDispatchFilter.java:207)
	at app//org.apache.solr.servlet.ServletUtils.traceHttpRequestExecution2(ServletUtils.java:257)
	at app//org.apache.solr.servlet.ServletUtils.rateLimitRequest(ServletUtils.java:227)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:202)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:184)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:187)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1434)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
	at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)
	at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1349)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
	at app//org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:713)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.server.Server.handle(Server.java:516)
	at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:400)
	at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:645)
	at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:392)
	at app//org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:352)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
	at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
Caused by: java.lang.AssertionError: Directory not closed: MockDirectoryWrapper(ByteBuffersDirectory@1ee8fda3 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@35462e1b)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.lucene.util.CloseableDirectory.close(CloseableDirectory.java:43)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:789)
	at com.carrotsearch.randomizedtesting.RandomizedContext.closeResources(RandomizedContext.java:225)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.afterAlways(RandomizedRunner.java:805)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	... 1 more


FAILED:  org.apache.solr.schema.TestManagedSchemaAPI.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=4078, name=qtp537313834-4078, state=RUNNABLE, group=TGRP-TestManagedSchemaAPI], registration stack trace below.

Stack Trace:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=4078, name=qtp537313834-4078, state=RUNNABLE, group=TGRP-TestManagedSchemaAPI], registration stack trace below.
	at java.base@17.0.2/java.lang.Thread.getStackTrace(Thread.java:1610)
	at app//com.carrotsearch.randomizedtesting.RandomizedContext.closeAtEnd(RandomizedContext.java:157)
	at app//org.apache.lucene.util.LuceneTestCase.closeAfterSuite(LuceneTestCase.java:798)
	at app//org.apache.lucene.util.LuceneTestCase.wrapDirectory(LuceneTestCase.java:1504)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1368)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1350)
	at app//org.apache.solr.core.MockDirectoryFactory.create(MockDirectoryFactory.java:51)
	at app//org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:368)
	at app//org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:108)
	at app//org.apache.solr.core.SolrCore.initIndex(SolrCore.java:895)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1123)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1048)
	at app//org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1562)
	at app//org.apache.solr.core.CoreContainer.create(CoreContainer.java:1428)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:389)
	at app//org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:422)
	at app//org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:205)
	at app//org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:207)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:941)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:893)
	at app//org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:584)
	at app//org.apache.solr.servlet.SolrDispatchFilter.dispatch(SolrDispatchFilter.java:239)
	at app//org.apache.solr.servlet.SolrDispatchFilter.lambda$doFilter$0(SolrDispatchFilter.java:207)
	at app//org.apache.solr.servlet.ServletUtils.traceHttpRequestExecution2(ServletUtils.java:257)
	at app//org.apache.solr.servlet.ServletUtils.rateLimitRequest(ServletUtils.java:227)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:202)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:184)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:187)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1434)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
	at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)
	at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1349)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
	at app//org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:713)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.server.Server.handle(Server.java:516)
	at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:400)
	at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:645)
	at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:392)
	at app//org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:352)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
	at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
Caused by: java.lang.AssertionError: Directory not closed: MockDirectoryWrapper(ByteBuffersDirectory@5207387a lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@31ad98fc)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.lucene.util.CloseableDirectory.close(CloseableDirectory.java:43)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:789)
	at com.carrotsearch.randomizedtesting.RandomizedContext.closeResources(RandomizedContext.java:225)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.afterAlways(RandomizedRunner.java:805)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	... 1 more


FAILED:  org.apache.solr.schema.TestManagedSchemaAPI.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.schema.TestManagedSchemaAPI: 
   1) Thread[id=4172, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestManagedSchemaAPI]
        at java.base@17.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:328)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.lookupZKManagedSchemaPath(ManagedIndexSchemaFactory.java:125)
        at app//org.apache.solr.schema.ZkIndexSchemaReader.<init>(ZkIndexSchemaReader.java:56)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.inform(ManagedIndexSchemaFactory.java:537)
        at app//org.apache.solr.core.SolrResourceLoader.inform(SolrResourceLoader.java:728)
        at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1155)
        at app//org.apache.solr.core.SolrCore.reload(SolrCore.java:780)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1857)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3394)
        at app//org.apache.solr.core.SolrCore$$Lambda$747/0x000000080121c000.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2761)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1197/0x000000080161fdd0.run(Unknown Source)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   2) Thread[id=4179, name=searcherExecutor-3734-thread-1-processing-127.0.0.1:59727_solr, state=WAITING, group=TGRP-TestManagedSchemaAPI]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
        at java.base@17.0.2/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:230)
        at app//org.apache.solr.core.SolrCore.lambda$new$2(SolrCore.java:1142)
        at app//org.apache.solr.core.SolrCore$$Lambda$724/0x00000008011ffdb0.call(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.FutureTask.run(FutureTask.java:264)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$264/0x0000000800f90d30.run(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.schema.TestManagedSchemaAPI: 
   1) Thread[id=4172, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestManagedSchemaAPI]
        at java.base@17.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:328)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.lookupZKManagedSchemaPath(ManagedIndexSchemaFactory.java:125)
        at app//org.apache.solr.schema.ZkIndexSchemaReader.<init>(ZkIndexSchemaReader.java:56)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.inform(ManagedIndexSchemaFactory.java:537)
        at app//org.apache.solr.core.SolrResourceLoader.inform(SolrResourceLoader.java:728)
        at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1155)
        at app//org.apache.solr.core.SolrCore.reload(SolrCore.java:780)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1857)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3394)
        at app//org.apache.solr.core.SolrCore$$Lambda$747/0x000000080121c000.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2761)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1197/0x000000080161fdd0.run(Unknown Source)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   2) Thread[id=4179, name=searcherExecutor-3734-thread-1-processing-127.0.0.1:59727_solr, state=WAITING, group=TGRP-TestManagedSchemaAPI]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
        at java.base@17.0.2/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:230)
        at app//org.apache.solr.core.SolrCore.lambda$new$2(SolrCore.java:1142)
        at app//org.apache.solr.core.SolrCore$$Lambda$724/0x00000008011ffdb0.call(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.FutureTask.run(FutureTask.java:264)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$264/0x0000000800f90d30.run(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
	at __randomizedtesting.SeedInfo.seed([86B33B7A150BDCB0]:0)


FAILED:  org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=1095, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@17.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:328)
        at app//org.apache.solr.cloud.ZkController.pathExists(ZkController.java:1228)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:73)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getSnowballWordSet(AbstractAnalysisFactory.java:321)
        at app//org.apache.lucene.analysis.en.AbstractWordsFileFilterFactory.inform(AbstractWordsFileFilterFactory.java:90)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:762)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1470)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1319)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1307)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:654)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:188)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:119)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:279)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:345)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:256)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$635/0x0000000801156378.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:49)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:252)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1850)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3394)
        at app//org.apache.solr.core.SolrCore$$Lambda$747/0x000000080121c000.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2761)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1197/0x000000080161fdd0.run(Unknown Source)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=1095, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@17.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:328)
        at app//org.apache.solr.cloud.ZkController.pathExists(ZkController.java:1228)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:73)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getSnowballWordSet(AbstractAnalysisFactory.java:321)
        at app//org.apache.lucene.analysis.en.AbstractWordsFileFilterFactory.inform(AbstractWordsFileFilterFactory.java:90)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:762)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1470)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1319)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1307)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:654)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:188)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:119)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:279)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:345)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:256)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$635/0x0000000801156378.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:49)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:252)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1850)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3394)
        at app//org.apache.solr.core.SolrCore$$Lambda$747/0x000000080121c000.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2761)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1197/0x000000080161fdd0.run(Unknown Source)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
	at __randomizedtesting.SeedInfo.seed([86B33B7A150BDCB0]:0)

[JENKINS] Solr-main-MacOSX (64bit/jdk-17.0.2) - Build # 871 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/871/
Java: 64bit/jdk-17.0.2 -XX:+UseCompressedOops -XX:+UseParallelGC

4 tests failed.
FAILED:  org.apache.solr.cloud.HttpPartitionOnCommitTest.test

Error Message:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:63978: Underlying core creation failed while creating collection: c8n_2x2_commits

Stack Trace:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:63978: Underlying core creation failed while creating collection: c8n_2x2_commits
	at __randomizedtesting.SeedInfo.seed([58B2CE9137865488:D0E6F14B997A3970]:0)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:715)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at app//org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1193)
	at app//org.apache.solr.cloud.AbstractBasicDistributedZkTestBase.createCollection(AbstractBasicDistributedZkTestBase.java:1177)
	at app//org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:2025)
	at app//org.apache.solr.cloud.HttpPartitionOnCommitTest.multiShardTest(HttpPartitionOnCommitTest.java:78)
	at app//org.apache.solr.cloud.HttpPartitionOnCommitTest.test(HttpPartitionOnCommitTest.java:69)
	at java.base@17.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base@17.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base@17.0.2/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base@17.0.2/java.lang.reflect.Method.invoke(Method.java:568)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1164)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1135)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)


FAILED:  org.apache.solr.cloud.MoveReplicaTest.test

Error Message:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:54159/solr: Underlying core creation failed while creating collection: MoveReplicaTest_coll_true

Stack Trace:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:54159/solr: Underlying core creation failed while creating collection: MoveReplicaTest_coll_true
	at __randomizedtesting.SeedInfo.seed([58B2CE9137865488:D0E6F14B997A3970]:0)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:715)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:405)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:371)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1248)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:969)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:896)
	at app//org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1193)
	at app//org.apache.solr.cloud.AbstractMoveReplicaTestBase.test(AbstractMoveReplicaTestBase.java:119)
	at app//org.apache.solr.cloud.MoveReplicaTest.test(MoveReplicaTest.java:30)
	at java.base@17.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base@17.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base@17.0.2/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base@17.0.2/java.lang.reflect.Method.invoke(Method.java:568)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)


FAILED:  org.apache.solr.cloud.OverseerTest.testReplay

Error Message:
java.util.concurrent.TimeoutException: Timeout waiting to see state for collection=collection1 :DocCollection(collection1//collections/collection1/state.json/2 )={
  "pullReplicas":"0",
  "replicationFactor":"1",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node1":{
          "core":"core1",
          "core_node_name":"core_node1",
          "roles":"",
          "node_name":"127.0.0.1:8983_solr",
          "base_url":"http://127.0.0.1:8983/solr",
          "state":"recovering",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT"},
        "core_node2":{
          "core":"core2",
          "core_node_name":"core_node2",
          "roles":"",
          "node_name":"node1:8983_",
          "base_url":"http://node1:8983",
          "state":"recovering",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"1",
  "tlogReplicas":"0",
  "autoCreated":"true"}

Stack Trace:
java.util.concurrent.TimeoutException: Timeout waiting to see state for collection=collection1 :DocCollection(collection1//collections/collection1/state.json/2 )={
  "pullReplicas":"0",
  "replicationFactor":"1",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node1":{
          "core":"core1",
          "core_node_name":"core_node1",
          "roles":"",
          "node_name":"127.0.0.1:8983_solr",
          "base_url":"http://127.0.0.1:8983/solr",
          "state":"recovering",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT"},
        "core_node2":{
          "core":"core2",
          "core_node_name":"core_node2",
          "roles":"",
          "node_name":"node1:8983_",
          "base_url":"http://node1:8983",
          "state":"recovering",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"1",
  "tlogReplicas":"0",
  "autoCreated":"true"}
	at __randomizedtesting.SeedInfo.seed([58B2CE9137865488:107BC6D5D9B525F3]:0)
	at org.apache.solr.common.cloud.ZkStateReader.waitForState(ZkStateReader.java:1781)
	at org.apache.solr.cloud.OverseerTest.testReplay(OverseerTest.java:1656)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:568)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base/java.lang.Thread.run(Thread.java:833)


FAILED:  org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=1061, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@17.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:361)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:75)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getLines(AbstractAnalysisFactory.java:302)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getWordSet(AbstractAnalysisFactory.java:293)
        at app//org.apache.lucene.analysis.en.AbstractWordsFileFilterFactory.inform(AbstractWordsFileFilterFactory.java:88)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:762)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1470)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1319)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1307)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:654)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:188)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:119)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:279)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:345)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:256)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$633/0x0000000801159138.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:49)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:252)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1850)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3394)
        at app//org.apache.solr.core.SolrCore$$Lambda$743/0x000000080121b918.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2761)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1185/0x00000008015ea1d8.run(Unknown Source)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=1061, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@17.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:361)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:75)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getLines(AbstractAnalysisFactory.java:302)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getWordSet(AbstractAnalysisFactory.java:293)
        at app//org.apache.lucene.analysis.en.AbstractWordsFileFilterFactory.inform(AbstractWordsFileFilterFactory.java:88)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:762)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1470)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1319)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1307)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:654)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:188)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:119)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:279)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:345)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:256)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$633/0x0000000801159138.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:49)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:252)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1850)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3394)
        at app//org.apache.solr.core.SolrCore$$Lambda$743/0x000000080121b918.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2761)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1185/0x00000008015ea1d8.run(Unknown Source)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
	at __randomizedtesting.SeedInfo.seed([58B2CE9137865488]:0)

[JENKINS] Solr-main-MacOSX (64bit/jdk-11.0.13) - Build # 870 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/870/
Java: 64bit/jdk-11.0.13 -XX:+UseCompressedOops -XX:+UseConcMarkSweepGC

2 tests failed.
FAILED:  org.apache.solr.cloud.ReindexCollectionTest.testAbort

Error Message:
java.lang.AssertionError: {responseHeader={status=0, QTime=37}, status={state=submitted, msg=found [19c98e31-c774-400b-8c68-e0fc75bd8daa] in submitted tasks}}

Stack Trace:
java.lang.AssertionError: {responseHeader={status=0, QTime=37}, status={state=submitted, msg=found [19c98e31-c774-400b-8c68-e0fc75bd8daa] in submitted tasks}}
	at __randomizedtesting.SeedInfo.seed([1FD87C90DC29EDFD:79BBCB07A8F37F84]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.junit.Assert.assertTrue(Assert.java:42)
	at org.junit.Assert.assertNotNull(Assert.java:713)
	at org.apache.solr.cloud.ReindexCollectionTest.testAbort(ReindexCollectionTest.java:439)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base/java.lang.Thread.run(Thread.java:829)


FAILED:  org.apache.solr.cloud.api.collections.CollectionsAPIDistributedZkTest.addReplicaTest

Error Message:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:53121/solr: Underlying core creation failed while creating collection: addReplicaColl

Stack Trace:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:53121/solr: Underlying core creation failed while creating collection: addReplicaColl
	at __randomizedtesting.SeedInfo.seed([1FD87C90DC29EDFD:8C782BB65221A62C]:0)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:715)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:405)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:371)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1248)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:969)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:896)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:223)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:238)
	at app//org.apache.solr.cloud.api.collections.AbstractCollectionsAPIDistributedZkTestBase.addReplicaTest(AbstractCollectionsAPIDistributedZkTestBase.java:648)
	at java.base@11.0.13/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base@11.0.13/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base@11.0.13/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base@11.0.13/java.lang.reflect.Method.invoke(Method.java:566)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)

[JENKINS] Solr-main-MacOSX (64bit/jdk-18) - Build # 869 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/869/
Java: 64bit/jdk-18 -XX:-UseCompressedOops -XX:+UseG1GC

6 tests failed.
FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test

Error Message:
org.apache.solr.cloud.ZkController$NotInClusterStateException: Number of replicas in the state does not match what we set:0 vs 3

Stack Trace:
org.apache.solr.cloud.ZkController$NotInClusterStateException: Number of replicas in the state does not match what we set:0 vs 3
	at __randomizedtesting.SeedInfo.seed([8D231CA2FCAC3F10:5772378525052E8]:0)
	at app//org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForActiveReplicaCount(AbstractFullDistribZkTestBase.java:709)
	at app//org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:639)
	at app//org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:378)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1162)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1135)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base@18/java.lang.Thread.run(Thread.java:833)


FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 83 threads leaked from SUITE scope at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest: 
   1) Thread[id=10819, name=zkConnectionManagerCallback-7358-thread-1, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@18/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3464)
        at java.base@18/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3435)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at java.base@18/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
   2) Thread[id=10777, name=qtp110215977-10777, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
   3) Thread[id=10912, name=zkCallback-7357-thread-8, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
   4) Thread[id=10791, name=zkConnectionManagerCallback-7339-thread-1, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@18/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3464)
        at java.base@18/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3435)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at java.base@18/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
   5) Thread[id=10884, name=updateExecutor-7352-thread-1-processing-127.0.0.1:53737_kfm collection1_shard2_replica_n5 collection1 shard2 core_node6, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
        at java.base@18/java.util.concurrent.FutureTask.awaitDone(FutureTask.java:447)
        at java.base@18/java.util.concurrent.FutureTask.get(FutureTask.java:190)
        at app//org.apache.solr.update.DefaultSolrCoreState$2.run(DefaultSolrCoreState.java:350)
        at app//com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:180)
        at java.base@18/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539)
        at java.base@18/java.util.concurrent.FutureTask.run(FutureTask.java:264)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$260/0x0000000800f6b520.run(Unknown Source)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
   6) Thread[id=10846, name=searcherExecutor-7366-thread-1, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@18/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3464)
        at java.base@18/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3435)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at java.base@18/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
   7) Thread[id=10863, name=zkCallback-7357-thread-3, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
   8) Thread[id=10744, name=qtp1804511539-10744-acceptor-0@46cdc323-ServerConnector@6fb1309a{HTTP/1.1, (http/1.1, h2c)}{127.0.0.1:53687}, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.Net.accept(Native Method)
        at java.base@18/sun.nio.ch.ServerSocketChannelImpl.implAccept(ServerSocketChannelImpl.java:425)
        at java.base@18/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:391)
        at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388)
        at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:702)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
   9) Thread[id=10804, name=qtp1740280584-10804, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  10) Thread[id=10869, name=zkCallback-7338-thread-3, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  11) Thread[id=10748, name=qtp1804511539-10748, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  12) Thread[id=10859, name=Connector-Scheduler-7c1fbca7-1, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at java.base@18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  13) Thread[id=10855, name=watches-7340-thread-1, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  14) Thread[id=10844, name=Connector-Scheduler-6480a6bb-1, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at java.base@18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  15) Thread[id=10792, name=zkCallback-7338-thread-1, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  16) Thread[id=10762, name=zkConnectionManagerCallback-7320-thread-1, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@18/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3464)
        at java.base@18/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3435)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at java.base@18/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  17) Thread[id=10813, name=h2sc-7353-thread-1, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@18/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$219/0x0000000800ec0530.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$260/0x0000000800f6b520.run(Unknown Source)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  18) Thread[id=10887, name=SocketProxy-Response-53687:53808, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.Net.poll(Native Method)
        at java.base@18/sun.nio.ch.NioSocketImpl.park(NioSocketImpl.java:178)
        at java.base@18/sun.nio.ch.NioSocketImpl.timedRead(NioSocketImpl.java:282)
        at java.base@18/sun.nio.ch.NioSocketImpl.implRead(NioSocketImpl.java:306)
        at java.base@18/sun.nio.ch.NioSocketImpl.read(NioSocketImpl.java:347)
        at java.base@18/sun.nio.ch.NioSocketImpl$1.read(NioSocketImpl.java:800)
        at java.base@18/java.net.Socket$SocketInputStream.read(Socket.java:966)
        at java.base@18/java.io.InputStream.read(InputStream.java:218)
        at app//org.apache.solr.client.solrj.cloud.SocketProxy$Bridge$Pump.run(SocketProxy.java:375)
  19) Thread[id=10799, name=qtp1740280584-10799, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@18/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$219/0x0000000800ec0530.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  20) Thread[id=10747, name=qtp1804511539-10747, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@18/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3464)
        at java.base@18/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3435)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at java.base@18/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@18/java.util.concurrent.ExecutorCompletionService.take(ExecutorCompletionService.java:200)
        at app//org.apache.solr.update.SolrCmdDistributor.blockAndDoRetries(SolrCmdDistributor.java:298)
        at app//org.apache.solr.update.SolrCmdDistributor.distribCommit(SolrCmdDistributor.java:279)
        at app//org.apache.solr.update.processor.DistributedZkUpdateProcessor.processCommit(DistributedZkUpdateProcessor.java:246)
        at app//org.apache.solr.update.processor.LogUpdateProcessorFactory$LogUpdateProcessor.processCommit(LogUpdateProcessorFactory.java:173)
        at app//org.apache.solr.handler.RequestHandlerUtils.handleCommit(RequestHandlerUtils.java:63)
        at app//org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:77)
        at app//org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:207)
        at app//org.apache.solr.core.SolrCore.execute(SolrCore.java:2866)
        at app//org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:887)
        at app//org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:606)
        at app//org.apache.solr.servlet.SolrDispatchFilter.dispatch(SolrDispatchFilter.java:239)
        at app//org.apache.solr.servlet.SolrDispatchFilter.lambda$doFilter$0(SolrDispatchFilter.java:207)
        at app//org.apache.solr.servlet.SolrDispatchFilter$$Lambda$381/0x000000080103f990.run(Unknown Source)
        at app//org.apache.solr.servlet.ServletUtils.traceHttpRequestExecution2(ServletUtils.java:257)
        at app//org.apache.solr.servlet.ServletUtils.rateLimitRequest(ServletUtils.java:227)
        at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:202)
        at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:184)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
        at app//org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:187)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
        at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
        at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
        at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1434)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
        at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)
        at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
        at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1349)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
        at app//org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
        at app//org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:713)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
        at app//org.eclipse.jetty.server.Server.handle(Server.java:516)
        at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:400)
        at app//org.eclipse.jetty.server.HttpChannel$$Lambda$379/0x0000000801039a00.dispatch(Unknown Source)
        at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:645)
        at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:392)
        at app//org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:277)
        at app//org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311)
        at app//org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:105)
        at app//org.eclipse.jetty.io.ChannelEndPoint$1.run(ChannelEndPoint.java:104)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  21) Thread[id=10862, name=zkCallback-7357-thread-2, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  22) Thread[id=10872, name=SocketProxy-Request-53806:53738, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.Net.poll(Native Method)
        at java.base@18/sun.nio.ch.NioSocketImpl.park(NioSocketImpl.java:178)
        at java.base@18/sun.nio.ch.NioSocketImpl.timedRead(NioSocketImpl.java:282)
        at java.base@18/sun.nio.ch.NioSocketImpl.implRead(NioSocketImpl.java:306)
        at java.base@18/sun.nio.ch.NioSocketImpl.read(NioSocketImpl.java:347)
        at java.base@18/sun.nio.ch.NioSocketImpl$1.read(NioSocketImpl.java:800)
        at java.base@18/java.net.Socket$SocketInputStream.read(Socket.java:966)
        at java.base@18/java.io.InputStream.read(InputStream.java:218)
        at app//org.apache.solr.client.solrj.cloud.SocketProxy$Bridge$Pump.run(SocketProxy.java:375)
  23) Thread[id=10875, name=watches-7321-thread-2-processing-127.0.0.1:53686_kfm collection1_shard2_replica_n1 collection1 shard2 core_node3, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkStateReader$CacheCleaner.run(ZkStateReader.java:2332)
        at java.base@18/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539)
        at java.base@18/java.util.concurrent.FutureTask.run(FutureTask.java:264)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$260/0x0000000800f6b520.run(Unknown Source)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  24) Thread[id=10743, name=qtp1804511539-10743, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@18/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$219/0x0000000800ec0530.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  25) Thread[id=10867, name=zkCallback-7338-thread-2, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  26) Thread[id=10756, name=h2sc-7315-thread-1, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@18/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$219/0x0000000800ec0530.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$260/0x0000000800f6b520.run(Unknown Source)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  27) Thread[id=10902, name=updateExecutor-7314-thread-1-processing-127.0.0.1:53686_kfm collection1_shard2_replica_n1 collection1 shard2 core_node3, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:717)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1074)
        at java.base@18/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:276)
        at app//org.eclipse.jetty.client.util.InputStreamResponseListener.get(InputStreamResponseListener.java:214)
        at app//org.apache.solr.client.solrj.impl.Http2SolrClient.request(Http2SolrClient.java:449)
        at app//org.apache.solr.client.solrj.impl.ConcurrentUpdateHttp2SolrClient.request(ConcurrentUpdateHttp2SolrClient.java:371)
        at app//org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1193)
        at app//org.apache.solr.update.SolrCmdDistributor.doRequest(SolrCmdDistributor.java:382)
        at app//org.apache.solr.update.SolrCmdDistributor.lambda$submit$0(SolrCmdDistributor.java:371)
        at app//org.apache.solr.update.SolrCmdDistributor$$Lambda$1086/0x00000008014a3520.call(Unknown Source)
        at java.base@18/java.util.concurrent.FutureTask.run(FutureTask.java:264)
        at java.base@18/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539)
        at java.base@18/java.util.concurrent.FutureTask.run(FutureTask.java:264)
        at app//com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:180)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$260/0x0000000800f6b520.run(Unknown Source)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  28) Thread[id=10913, name=zkCallback-7319-thread-3, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  29) Thread[id=10798, name=qtp1740280584-10798, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@18/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$219/0x0000000800ec0530.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  30) Thread[id=10772, name=qtp110215977-10772, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@18/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$219/0x0000000800ec0530.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  31) Thread[id=10810, name=httpShardExecutor-7351-thread-1, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@18/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$219/0x0000000800ec0530.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$260/0x0000000800f6b520.run(Unknown Source)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  32) Thread[id=10906, name=SocketProxy-Request-53810:53703, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.Net.poll(Native Method)
        at java.base@18/sun.nio.ch.NioSocketImpl.park(NioSocketImpl.java:178)
        at java.base@18/sun.nio.ch.NioSocketImpl.timedRead(NioSocketImpl.java:282)
        at java.base@18/sun.nio.ch.NioSocketImpl.implRead(NioSocketImpl.java:306)
        at java.base@18/sun.nio.ch.NioSocketImpl.read(NioSocketImpl.java:347)
        at java.base@18/sun.nio.ch.NioSocketImpl$1.read(NioSocketImpl.java:800)
        at java.base@18/java.net.Socket$SocketInputStream.read(Socket.java:966)
        at java.base@18/java.io.InputStream.read(InputStream.java:218)
        at app//org.apache.solr.client.solrj.cloud.SocketProxy$Bridge$Pump.run(SocketProxy.java:375)
  33) Thread[id=10915, name=zkCallback-7338-thread-5, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  34) Thread[id=10745, name=qtp1804511539-10745, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  35) Thread[id=10908, name=zkCallback-7357-thread-4, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  36) Thread[id=10914, name=zkCallback-7338-thread-4, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  37) Thread[id=10767, name=SocketProxy-Acceptor-53686, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.Net.poll(Native Method)
        at java.base@18/sun.nio.ch.NioSocketImpl.park(NioSocketImpl.java:178)
        at java.base@18/sun.nio.ch.NioSocketImpl.timedAccept(NioSocketImpl.java:707)
        at java.base@18/sun.nio.ch.NioSocketImpl.accept(NioSocketImpl.java:749)
        at java.base@18/java.net.ServerSocket.implAccept(ServerSocket.java:673)
        at java.base@18/java.net.ServerSocket.platformImplAccept(ServerSocket.java:639)
        at java.base@18/java.net.ServerSocket.implAccept(ServerSocket.java:615)
        at java.base@18/java.net.ServerSocket.implAccept(ServerSocket.java:572)
        at java.base@18/java.net.ServerSocket.accept(ServerSocket.java:530)
        at app//org.apache.solr.client.solrj.cloud.SocketProxy$Acceptor.run(SocketProxy.java:445)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  38) Thread[id=10907, name=SocketProxy-Response-53703:53810, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.Net.poll(Native Method)
        at java.base@18/sun.nio.ch.NioSocketImpl.park(NioSocketImpl.java:178)
        at java.base@18/sun.nio.ch.NioSocketImpl.timedRead(NioSocketImpl.java:282)
        at java.base@18/sun.nio.ch.NioSocketImpl.implRead(NioSocketImpl.java:306)
        at java.base@18/sun.nio.ch.NioSocketImpl.read(NioSocketImpl.java:347)
        at java.base@18/sun.nio.ch.NioSocketImpl$1.read(NioSocketImpl.java:800)
        at java.base@18/java.net.Socket$SocketInputStream.read(Socket.java:966)
        at java.base@18/java.io.InputStream.read(InputStream.java:218)
        at app//org.apache.solr.client.solrj.cloud.SocketProxy$Bridge$Pump.run(SocketProxy.java:375)
  39) Thread[id=10886, name=SocketProxy-Request-53808:53687, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.Net.poll(Native Method)
        at java.base@18/sun.nio.ch.NioSocketImpl.park(NioSocketImpl.java:178)
        at java.base@18/sun.nio.ch.NioSocketImpl.timedRead(NioSocketImpl.java:282)
        at java.base@18/sun.nio.ch.NioSocketImpl.implRead(NioSocketImpl.java:306)
        at java.base@18/sun.nio.ch.NioSocketImpl.read(NioSocketImpl.java:347)
        at java.base@18/sun.nio.ch.NioSocketImpl$1.read(NioSocketImpl.java:800)
        at java.base@18/java.net.Socket$SocketInputStream.read(Socket.java:966)
        at java.base@18/java.io.InputStream.read(InputStream.java:218)
        at app//org.apache.solr.client.solrj.cloud.SocketProxy$Bridge$Pump.run(SocketProxy.java:375)
  40) Thread[id=10821, name=CloserThread, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/java.lang.Object.wait(Native Method)
        at java.base@18/java.lang.Object.wait(Object.java:338)
        at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2409)
  41) Thread[id=10909, name=zkCallback-7357-thread-5, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  42) Thread[id=10805, name=qtp1740280584-10805, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  43) Thread[id=10874, name=zkCallback-7319-thread-2, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  44) Thread[id=10806, name=Session-HouseKeeper-5502908a-1, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at java.base@18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  45) Thread[id=10885, name=recoveryExecutor-7354-thread-1-processing-127.0.0.1:53737_kfm collection1_shard2_replica_n5 collection1 shard2 core_node6, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.Net.poll(Native Method)
        at java.base@18/sun.nio.ch.NioSocketImpl.park(NioSocketImpl.java:178)
        at java.base@18/sun.nio.ch.NioSocketImpl.timedRead(NioSocketImpl.java:282)
        at java.base@18/sun.nio.ch.NioSocketImpl.implRead(NioSocketImpl.java:306)
        at java.base@18/sun.nio.ch.NioSocketImpl.read(NioSocketImpl.java:347)
        at java.base@18/sun.nio.ch.NioSocketImpl$1.read(NioSocketImpl.java:800)
        at java.base@18/java.net.Socket$SocketInputStream.read(Socket.java:966)
        at app//org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
        at app//org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
        at app//org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:280)
        at app//org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
        at app//org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
        at app//org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
        at app//org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
        at app//org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:157)
        at app//org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
        at app//org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
        at app//org.apache.solr.util.stats.InstrumentedHttpRequestExecutor.execute(InstrumentedHttpRequestExecutor.java:142)
        at app//org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
        at app//org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186)
        at app//org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
        at app//org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
        at app//org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
        at app//org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
        at app//org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
        at app//org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:576)
        at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
        at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
        at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:223)
        at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:238)
        at app//org.apache.solr.cloud.RecoveryStrategy.commitOnLeader(RecoveryStrategy.java:289)
        at app//org.apache.solr.cloud.RecoveryStrategy.replicate(RecoveryStrategy.java:226)
        at app//org.apache.solr.cloud.RecoveryStrategy.doSyncOrReplicateRecovery(RecoveryStrategy.java:700)
        at app//org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:332)
        at app//org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:308)
        at app//com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:180)
        at java.base@18/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539)
        at java.base@18/java.util.concurrent.FutureTask.run(FutureTask.java:264)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$260/0x0000000800f6b520.run(Unknown Source)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  46) Thread[id=10753, name=httpShardExecutor-7313-thread-1, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@18/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$219/0x0000000800ec0530.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$260/0x0000000800f6b520.run(Unknown Source)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  47) Thread[id=10765, name=CloserThread, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/java.lang.Object.wait(Native Method)
        at java.base@18/java.lang.Object.wait(Object.java:338)
        at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2409)
  48) Thread[id=10800, name=qtp1740280584-10800, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@18/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$219/0x0000000800ec0530.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  49) Thread[id=10905, name=h2sc-7315-thread-3, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@18/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3464)
        at java.base@18/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3435)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  50) Thread[id=10865, name=searcherExecutor-7375-thread-1, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@18/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3464)
        at java.base@18/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3435)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at java.base@18/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  51) Thread[id=10903, name=HttpClient@7841103c-scheduler-1, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at java.base@18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  52) Thread[id=10773, name=qtp110215977-10773-acceptor-0@7fe48efd-ServerConnector@6480a6bb{HTTP/1.1, (http/1.1, h2c)}{127.0.0.1:53703}, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.Net.accept(Native Method)
        at java.base@18/sun.nio.ch.ServerSocketChannelImpl.implAccept(ServerSocketChannelImpl.java:425)
        at java.base@18/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:391)
        at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388)
        at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:702)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  53) Thread[id=10910, name=zkCallback-7357-thread-6, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  54) Thread[id=10774, name=qtp110215977-10774, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:361)
        at app//org.apache.solr.common.cloud.ZkStateReader.fetchCollectionProperties(ZkStateReader.java:1195)
        at app//org.apache.solr.common.cloud.ZkStateReader.getCollectionProperties(ZkStateReader.java:1142)
        at app//org.apache.solr.update.processor.RoutedAliasUpdateProcessor.wrap(RoutedAliasUpdateProcessor.java:104)
        at app//org.apache.solr.update.processor.DistributedUpdateProcessorFactory.getInstance(DistributedUpdateProcessorFactory.java:62)
        at app//org.apache.solr.update.processor.UpdateRequestProcessorChain.createProcessor(UpdateRequestProcessorChain.java:243)
        at app//org.apache.solr.update.processor.UpdateRequestProcessorChain.createProcessor(UpdateRequestProcessorChain.java:215)
        at app//org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:70)
        at app//org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:207)
        at app//org.apache.solr.core.SolrCore.execute(SolrCore.java:2866)
        at app//org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:887)
        at app//org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:606)
        at app//org.apache.solr.servlet.SolrDispatchFilter.dispatch(SolrDispatchFilter.java:239)
        at app//org.apache.solr.servlet.SolrDispatchFilter.lambda$doFilter$0(SolrDispatchFilter.java:207)
        at app//org.apache.solr.servlet.SolrDispatchFilter$$Lambda$381/0x000000080103f990.run(Unknown Source)
        at app//org.apache.solr.servlet.ServletUtils.traceHttpRequestExecution2(ServletUtils.java:257)
        at app//org.apache.solr.servlet.ServletUtils.rateLimitRequest(ServletUtils.java:227)
        at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:202)
        at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:184)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
        at app//org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:187)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
        at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
        at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
        at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1434)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
        at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)
        at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
        at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1349)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
        at app//org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
        at app//org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:713)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
        at app//org.eclipse.jetty.server.Server.handle(Server.java:516)
        at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:400)
        at app//org.eclipse.jetty.server.HttpChannel$$Lambda$379/0x0000000801039a00.dispatch(Unknown Source)
        at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:645)
        at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:392)
        at app//org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:352)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  55) Thread[id=10771, name=qtp110215977-10771, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@18/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$219/0x0000000800ec0530.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  56) Thread[id=10801, name=qtp1740280584-10801-acceptor-0@5910fe5b-ServerConnector@7c1fbca7{HTTP/1.1, (http/1.1, h2c)}{127.0.0.1:53738}, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.Net.accept(Native Method)
        at java.base@18/sun.nio.ch.ServerSocketChannelImpl.implAccept(ServerSocketChannelImpl.java:425)
        at java.base@18/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:391)
        at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388)
        at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:702)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  57) Thread[id=10868, name=searcherExecutor-7381-thread-1, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@18/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3464)
        at java.base@18/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3435)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at java.base@18/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  58) Thread[id=10741, name=qtp1804511539-10741, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@18/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$219/0x0000000800ec0530.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  59) Thread[id=10838, name=Connector-Scheduler-6fb1309a-1, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at java.base@18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  60) Thread[id=10782, name=httpShardExecutor-7332-thread-1, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@18/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$219/0x0000000800ec0530.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$260/0x0000000800f6b520.run(Unknown Source)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  61) Thread[id=10776, name=qtp110215977-10776, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  62) Thread[id=10770, name=qtp110215977-10770, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@18/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$219/0x0000000800ec0530.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  63) Thread[id=10749, name=Session-HouseKeeper-3a06ffc4-1, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at java.base@18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  64) Thread[id=10873, name=SocketProxy-Response-53738:53806, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.Net.poll(Native Method)
        at java.base@18/sun.nio.ch.NioSocketImpl.park(NioSocketImpl.java:178)
        at java.base@18/sun.nio.ch.NioSocketImpl.timedRead(NioSocketImpl.java:282)
        at java.base@18/sun.nio.ch.NioSocketImpl.implRead(NioSocketImpl.java:306)
        at java.base@18/sun.nio.ch.NioSocketImpl.read(NioSocketImpl.java:347)
        at java.base@18/sun.nio.ch.NioSocketImpl$1.read(NioSocketImpl.java:800)
        at java.base@18/java.net.Socket$SocketInputStream.read(Socket.java:966)
        at java.base@18/java.io.InputStream.read(InputStream.java:218)
        at app//org.apache.solr.client.solrj.cloud.SocketProxy$Bridge$Pump.run(SocketProxy.java:375)
  65) Thread[id=10911, name=zkCallback-7357-thread-7, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  66) Thread[id=10742, name=qtp1804511539-10742, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@18/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$219/0x0000000800ec0530.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  67) Thread[id=10778, name=Session-HouseKeeper-377197e3-1, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at java.base@18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  68) Thread[id=10870, name=HttpClient@7dc56056-scheduler-1, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at java.base@18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  69) Thread[id=10840, name=watches-7321-thread-1, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  70) Thread[id=10793, name=CloserThread, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/java.lang.Object.wait(Native Method)
        at java.base@18/java.lang.Object.wait(Object.java:338)
        at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2409)
  71) Thread[id=10904, name=h2sc-7315-thread-2, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@18/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3464)
        at java.base@18/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3435)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  72) Thread[id=10785, name=h2sc-7334-thread-1, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@18/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$219/0x0000000800ec0530.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$260/0x0000000800f6b520.run(Unknown Source)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  73) Thread[id=10823, name=SocketProxy-Acceptor-53737, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.Net.poll(Native Method)
        at java.base@18/sun.nio.ch.NioSocketImpl.park(NioSocketImpl.java:178)
        at java.base@18/sun.nio.ch.NioSocketImpl.timedAccept(NioSocketImpl.java:707)
        at java.base@18/sun.nio.ch.NioSocketImpl.accept(NioSocketImpl.java:749)
        at java.base@18/java.net.ServerSocket.implAccept(ServerSocket.java:673)
        at java.base@18/java.net.ServerSocket.platformImplAccept(ServerSocket.java:639)
        at java.base@18/java.net.ServerSocket.implAccept(ServerSocket.java:615)
        at java.base@18/java.net.ServerSocket.implAccept(ServerSocket.java:572)
        at java.base@18/java.net.ServerSocket.accept(ServerSocket.java:530)
        at app//org.apache.solr.client.solrj.cloud.SocketProxy$Acceptor.run(SocketProxy.java:445)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  74) Thread[id=10861, name=watches-7359-thread-1, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  75) Thread[id=10795, name=SocketProxy-Acceptor-53702, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.Net.poll(Native Method)
        at java.base@18/sun.nio.ch.NioSocketImpl.park(NioSocketImpl.java:178)
        at java.base@18/sun.nio.ch.NioSocketImpl.timedAccept(NioSocketImpl.java:707)
        at java.base@18/sun.nio.ch.NioSocketImpl.accept(NioSocketImpl.java:749)
        at java.base@18/java.net.ServerSocket.implAccept(ServerSocket.java:673)
        at java.base@18/java.net.ServerSocket.platformImplAccept(ServerSocket.java:639)
        at java.base@18/java.net.ServerSocket.implAccept(ServerSocket.java:615)
        at java.base@18/java.net.ServerSocket.implAccept(ServerSocket.java:572)
        at java.base@18/java.net.ServerSocket.accept(ServerSocket.java:530)
        at app//org.apache.solr.client.solrj.cloud.SocketProxy$Acceptor.run(SocketProxy.java:445)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  76) Thread[id=10802, name=qtp1740280584-10802, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  77) Thread[id=10916, name=zkCallback-7319-thread-4, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  78) Thread[id=10764, name=zkCallback-7319-thread-1, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:328)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.ensureExists(ZkCmdExecutor.java:145)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.ensureExists(ZkCmdExecutor.java:104)
        at app//org.apache.solr.cloud.DistributedMap.<init>(DistributedMap.java:47)
        at app//org.apache.solr.cloud.SizeLimitedDistributedMap.<init>(SizeLimitedDistributedMap.java:47)
        at app//org.apache.solr.cloud.Overseer.getFailureMap(Overseer.java:1062)
        at app//org.apache.solr.cloud.Overseer$ClusterStateUpdater.<init>(Overseer.java:207)
        at app//org.apache.solr.cloud.Overseer.start(Overseer.java:729)
        at app//org.apache.solr.cloud.OverseerElectionContext.runLeaderProcess(OverseerElectionContext.java:77)
        at app//org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:175)
        at app//org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135)
        at app//org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:355)
        at app//org.apache.solr.common.cloud.SolrZkClient$ProcessWatchWithExecutor.lambda$process$1(SolrZkClient.java:922)
        at app//org.apache.solr.common.cloud.SolrZkClient$ProcessWatchWithExecutor$$Lambda$490/0x00000008010d4000.run(Unknown Source)
        at java.base@18/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539)
        at java.base@18/java.util.concurrent.FutureTask.run(FutureTask.java:264)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$260/0x0000000800f6b520.run(Unknown Source)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  79) Thread[id=10803, name=qtp1740280584-10803, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  80) Thread[id=10890, name=watches-7340-thread-2-processing-127.0.0.1:53702_kfm collection1_shard1_replica_n2 collection1 shard1 core_node4, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkStateReader$CacheCleaner.run(ZkStateReader.java:2332)
        at java.base@18/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539)
        at java.base@18/java.util.concurrent.FutureTask.run(FutureTask.java:264)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$260/0x0000000800f6b520.run(Unknown Source)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  81) Thread[id=10775, name=qtp110215977-10775, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  82) Thread[id=10746, name=qtp1804511539-10746, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@18/java.lang.Thread.run(Thread.java:833)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 83 threads leaked from SUITE scope at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest: 
   1) Thread[id=10819, name=zkConnectionManagerCallback-7358-thread-1, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@18/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3464)
        at java.base@18/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3435)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at java.base@18/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
   2) Thread[id=10777, name=qtp110215977-10777, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
   3) Thread[id=10912, name=zkCallback-7357-thread-8, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
   4) Thread[id=10791, name=zkConnectionManagerCallback-7339-thread-1, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@18/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3464)
        at java.base@18/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3435)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at java.base@18/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
   5) Thread[id=10884, name=updateExecutor-7352-thread-1-processing-127.0.0.1:53737_kfm collection1_shard2_replica_n5 collection1 shard2 core_node6, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
        at java.base@18/java.util.concurrent.FutureTask.awaitDone(FutureTask.java:447)
        at java.base@18/java.util.concurrent.FutureTask.get(FutureTask.java:190)
        at app//org.apache.solr.update.DefaultSolrCoreState$2.run(DefaultSolrCoreState.java:350)
        at app//com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:180)
        at java.base@18/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539)
        at java.base@18/java.util.concurrent.FutureTask.run(FutureTask.java:264)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$260/0x0000000800f6b520.run(Unknown Source)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
   6) Thread[id=10846, name=searcherExecutor-7366-thread-1, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@18/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3464)
        at java.base@18/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3435)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at java.base@18/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
   7) Thread[id=10863, name=zkCallback-7357-thread-3, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
   8) Thread[id=10744, name=qtp1804511539-10744-acceptor-0@46cdc323-ServerConnector@6fb1309a{HTTP/1.1, (http/1.1, h2c)}{127.0.0.1:53687}, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.Net.accept(Native Method)
        at java.base@18/sun.nio.ch.ServerSocketChannelImpl.implAccept(ServerSocketChannelImpl.java:425)
        at java.base@18/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:391)
        at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388)
        at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:702)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
   9) Thread[id=10804, name=qtp1740280584-10804, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  10) Thread[id=10869, name=zkCallback-7338-thread-3, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  11) Thread[id=10748, name=qtp1804511539-10748, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  12) Thread[id=10859, name=Connector-Scheduler-7c1fbca7-1, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at java.base@18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  13) Thread[id=10855, name=watches-7340-thread-1, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  14) Thread[id=10844, name=Connector-Scheduler-6480a6bb-1, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at java.base@18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  15) Thread[id=10792, name=zkCallback-7338-thread-1, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  16) Thread[id=10762, name=zkConnectionManagerCallback-7320-thread-1, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@18/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3464)
        at java.base@18/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3435)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at java.base@18/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  17) Thread[id=10813, name=h2sc-7353-thread-1, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@18/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$219/0x0000000800ec0530.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$260/0x0000000800f6b520.run(Unknown Source)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  18) Thread[id=10887, name=SocketProxy-Response-53687:53808, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.Net.poll(Native Method)
        at java.base@18/sun.nio.ch.NioSocketImpl.park(NioSocketImpl.java:178)
        at java.base@18/sun.nio.ch.NioSocketImpl.timedRead(NioSocketImpl.java:282)
        at java.base@18/sun.nio.ch.NioSocketImpl.implRead(NioSocketImpl.java:306)
        at java.base@18/sun.nio.ch.NioSocketImpl.read(NioSocketImpl.java:347)
        at java.base@18/sun.nio.ch.NioSocketImpl$1.read(NioSocketImpl.java:800)
        at java.base@18/java.net.Socket$SocketInputStream.read(Socket.java:966)
        at java.base@18/java.io.InputStream.read(InputStream.java:218)
        at app//org.apache.solr.client.solrj.cloud.SocketProxy$Bridge$Pump.run(SocketProxy.java:375)
  19) Thread[id=10799, name=qtp1740280584-10799, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@18/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$219/0x0000000800ec0530.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  20) Thread[id=10747, name=qtp1804511539-10747, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@18/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3464)
        at java.base@18/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3435)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at java.base@18/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@18/java.util.concurrent.ExecutorCompletionService.take(ExecutorCompletionService.java:200)
        at app//org.apache.solr.update.SolrCmdDistributor.blockAndDoRetries(SolrCmdDistributor.java:298)
        at app//org.apache.solr.update.SolrCmdDistributor.distribCommit(SolrCmdDistributor.java:279)
        at app//org.apache.solr.update.processor.DistributedZkUpdateProcessor.processCommit(DistributedZkUpdateProcessor.java:246)
        at app//org.apache.solr.update.processor.LogUpdateProcessorFactory$LogUpdateProcessor.processCommit(LogUpdateProcessorFactory.java:173)
        at app//org.apache.solr.handler.RequestHandlerUtils.handleCommit(RequestHandlerUtils.java:63)
        at app//org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:77)
        at app//org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:207)
        at app//org.apache.solr.core.SolrCore.execute(SolrCore.java:2866)
        at app//org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:887)
        at app//org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:606)
        at app//org.apache.solr.servlet.SolrDispatchFilter.dispatch(SolrDispatchFilter.java:239)
        at app//org.apache.solr.servlet.SolrDispatchFilter.lambda$doFilter$0(SolrDispatchFilter.java:207)
        at app//org.apache.solr.servlet.SolrDispatchFilter$$Lambda$381/0x000000080103f990.run(Unknown Source)
        at app//org.apache.solr.servlet.ServletUtils.traceHttpRequestExecution2(ServletUtils.java:257)
        at app//org.apache.solr.servlet.ServletUtils.rateLimitRequest(ServletUtils.java:227)
        at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:202)
        at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:184)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
        at app//org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:187)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
        at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
        at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
        at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1434)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
        at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)
        at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
        at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1349)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
        at app//org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
        at app//org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:713)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
        at app//org.eclipse.jetty.server.Server.handle(Server.java:516)
        at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:400)
        at app//org.eclipse.jetty.server.HttpChannel$$Lambda$379/0x0000000801039a00.dispatch(Unknown Source)
        at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:645)
        at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:392)
        at app//org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:277)
        at app//org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311)
        at app//org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:105)
        at app//org.eclipse.jetty.io.ChannelEndPoint$1.run(ChannelEndPoint.java:104)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  21) Thread[id=10862, name=zkCallback-7357-thread-2, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  22) Thread[id=10872, name=SocketProxy-Request-53806:53738, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.Net.poll(Native Method)
        at java.base@18/sun.nio.ch.NioSocketImpl.park(NioSocketImpl.java:178)
        at java.base@18/sun.nio.ch.NioSocketImpl.timedRead(NioSocketImpl.java:282)
        at java.base@18/sun.nio.ch.NioSocketImpl.implRead(NioSocketImpl.java:306)
        at java.base@18/sun.nio.ch.NioSocketImpl.read(NioSocketImpl.java:347)
        at java.base@18/sun.nio.ch.NioSocketImpl$1.read(NioSocketImpl.java:800)
        at java.base@18/java.net.Socket$SocketInputStream.read(Socket.java:966)
        at java.base@18/java.io.InputStream.read(InputStream.java:218)
        at app//org.apache.solr.client.solrj.cloud.SocketProxy$Bridge$Pump.run(SocketProxy.java:375)
  23) Thread[id=10875, name=watches-7321-thread-2-processing-127.0.0.1:53686_kfm collection1_shard2_replica_n1 collection1 shard2 core_node3, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkStateReader$CacheCleaner.run(ZkStateReader.java:2332)
        at java.base@18/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539)
        at java.base@18/java.util.concurrent.FutureTask.run(FutureTask.java:264)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$260/0x0000000800f6b520.run(Unknown Source)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  24) Thread[id=10743, name=qtp1804511539-10743, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@18/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$219/0x0000000800ec0530.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  25) Thread[id=10867, name=zkCallback-7338-thread-2, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  26) Thread[id=10756, name=h2sc-7315-thread-1, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@18/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$219/0x0000000800ec0530.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$260/0x0000000800f6b520.run(Unknown Source)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  27) Thread[id=10902, name=updateExecutor-7314-thread-1-processing-127.0.0.1:53686_kfm collection1_shard2_replica_n1 collection1 shard2 core_node3, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:717)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1074)
        at java.base@18/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:276)
        at app//org.eclipse.jetty.client.util.InputStreamResponseListener.get(InputStreamResponseListener.java:214)
        at app//org.apache.solr.client.solrj.impl.Http2SolrClient.request(Http2SolrClient.java:449)
        at app//org.apache.solr.client.solrj.impl.ConcurrentUpdateHttp2SolrClient.request(ConcurrentUpdateHttp2SolrClient.java:371)
        at app//org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1193)
        at app//org.apache.solr.update.SolrCmdDistributor.doRequest(SolrCmdDistributor.java:382)
        at app//org.apache.solr.update.SolrCmdDistributor.lambda$submit$0(SolrCmdDistributor.java:371)
        at app//org.apache.solr.update.SolrCmdDistributor$$Lambda$1086/0x00000008014a3520.call(Unknown Source)
        at java.base@18/java.util.concurrent.FutureTask.run(FutureTask.java:264)
        at java.base@18/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539)
        at java.base@18/java.util.concurrent.FutureTask.run(FutureTask.java:264)
        at app//com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:180)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$260/0x0000000800f6b520.run(Unknown Source)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  28) Thread[id=10913, name=zkCallback-7319-thread-3, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  29) Thread[id=10798, name=qtp1740280584-10798, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@18/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$219/0x0000000800ec0530.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  30) Thread[id=10772, name=qtp110215977-10772, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@18/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$219/0x0000000800ec0530.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  31) Thread[id=10810, name=httpShardExecutor-7351-thread-1, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@18/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$219/0x0000000800ec0530.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$260/0x0000000800f6b520.run(Unknown Source)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  32) Thread[id=10906, name=SocketProxy-Request-53810:53703, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.Net.poll(Native Method)
        at java.base@18/sun.nio.ch.NioSocketImpl.park(NioSocketImpl.java:178)
        at java.base@18/sun.nio.ch.NioSocketImpl.timedRead(NioSocketImpl.java:282)
        at java.base@18/sun.nio.ch.NioSocketImpl.implRead(NioSocketImpl.java:306)
        at java.base@18/sun.nio.ch.NioSocketImpl.read(NioSocketImpl.java:347)
        at java.base@18/sun.nio.ch.NioSocketImpl$1.read(NioSocketImpl.java:800)
        at java.base@18/java.net.Socket$SocketInputStream.read(Socket.java:966)
        at java.base@18/java.io.InputStream.read(InputStream.java:218)
        at app//org.apache.solr.client.solrj.cloud.SocketProxy$Bridge$Pump.run(SocketProxy.java:375)
  33) Thread[id=10915, name=zkCallback-7338-thread-5, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  34) Thread[id=10745, name=qtp1804511539-10745, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  35) Thread[id=10908, name=zkCallback-7357-thread-4, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  36) Thread[id=10914, name=zkCallback-7338-thread-4, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  37) Thread[id=10767, name=SocketProxy-Acceptor-53686, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.Net.poll(Native Method)
        at java.base@18/sun.nio.ch.NioSocketImpl.park(NioSocketImpl.java:178)
        at java.base@18/sun.nio.ch.NioSocketImpl.timedAccept(NioSocketImpl.java:707)
        at java.base@18/sun.nio.ch.NioSocketImpl.accept(NioSocketImpl.java:749)
        at java.base@18/java.net.ServerSocket.implAccept(ServerSocket.java:673)
        at java.base@18/java.net.ServerSocket.platformImplAccept(ServerSocket.java:639)
        at java.base@18/java.net.ServerSocket.implAccept(ServerSocket.java:615)
        at java.base@18/java.net.ServerSocket.implAccept(ServerSocket.java:572)
        at java.base@18/java.net.ServerSocket.accept(ServerSocket.java:530)
        at app//org.apache.solr.client.solrj.cloud.SocketProxy$Acceptor.run(SocketProxy.java:445)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  38) Thread[id=10907, name=SocketProxy-Response-53703:53810, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.Net.poll(Native Method)
        at java.base@18/sun.nio.ch.NioSocketImpl.park(NioSocketImpl.java:178)
        at java.base@18/sun.nio.ch.NioSocketImpl.timedRead(NioSocketImpl.java:282)
        at java.base@18/sun.nio.ch.NioSocketImpl.implRead(NioSocketImpl.java:306)
        at java.base@18/sun.nio.ch.NioSocketImpl.read(NioSocketImpl.java:347)
        at java.base@18/sun.nio.ch.NioSocketImpl$1.read(NioSocketImpl.java:800)
        at java.base@18/java.net.Socket$SocketInputStream.read(Socket.java:966)
        at java.base@18/java.io.InputStream.read(InputStream.java:218)
        at app//org.apache.solr.client.solrj.cloud.SocketProxy$Bridge$Pump.run(SocketProxy.java:375)
  39) Thread[id=10886, name=SocketProxy-Request-53808:53687, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.Net.poll(Native Method)
        at java.base@18/sun.nio.ch.NioSocketImpl.park(NioSocketImpl.java:178)
        at java.base@18/sun.nio.ch.NioSocketImpl.timedRead(NioSocketImpl.java:282)
        at java.base@18/sun.nio.ch.NioSocketImpl.implRead(NioSocketImpl.java:306)
        at java.base@18/sun.nio.ch.NioSocketImpl.read(NioSocketImpl.java:347)
        at java.base@18/sun.nio.ch.NioSocketImpl$1.read(NioSocketImpl.java:800)
        at java.base@18/java.net.Socket$SocketInputStream.read(Socket.java:966)
        at java.base@18/java.io.InputStream.read(InputStream.java:218)
        at app//org.apache.solr.client.solrj.cloud.SocketProxy$Bridge$Pump.run(SocketProxy.java:375)
  40) Thread[id=10821, name=CloserThread, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/java.lang.Object.wait(Native Method)
        at java.base@18/java.lang.Object.wait(Object.java:338)
        at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2409)
  41) Thread[id=10909, name=zkCallback-7357-thread-5, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  42) Thread[id=10805, name=qtp1740280584-10805, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  43) Thread[id=10874, name=zkCallback-7319-thread-2, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  44) Thread[id=10806, name=Session-HouseKeeper-5502908a-1, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at java.base@18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  45) Thread[id=10885, name=recoveryExecutor-7354-thread-1-processing-127.0.0.1:53737_kfm collection1_shard2_replica_n5 collection1 shard2 core_node6, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.Net.poll(Native Method)
        at java.base@18/sun.nio.ch.NioSocketImpl.park(NioSocketImpl.java:178)
        at java.base@18/sun.nio.ch.NioSocketImpl.timedRead(NioSocketImpl.java:282)
        at java.base@18/sun.nio.ch.NioSocketImpl.implRead(NioSocketImpl.java:306)
        at java.base@18/sun.nio.ch.NioSocketImpl.read(NioSocketImpl.java:347)
        at java.base@18/sun.nio.ch.NioSocketImpl$1.read(NioSocketImpl.java:800)
        at java.base@18/java.net.Socket$SocketInputStream.read(Socket.java:966)
        at app//org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
        at app//org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
        at app//org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:280)
        at app//org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
        at app//org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
        at app//org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
        at app//org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
        at app//org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:157)
        at app//org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
        at app//org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
        at app//org.apache.solr.util.stats.InstrumentedHttpRequestExecutor.execute(InstrumentedHttpRequestExecutor.java:142)
        at app//org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
        at app//org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186)
        at app//org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
        at app//org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
        at app//org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
        at app//org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
        at app//org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
        at app//org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:576)
        at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
        at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
        at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:223)
        at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:238)
        at app//org.apache.solr.cloud.RecoveryStrategy.commitOnLeader(RecoveryStrategy.java:289)
        at app//org.apache.solr.cloud.RecoveryStrategy.replicate(RecoveryStrategy.java:226)
        at app//org.apache.solr.cloud.RecoveryStrategy.doSyncOrReplicateRecovery(RecoveryStrategy.java:700)
        at app//org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:332)
        at app//org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:308)
        at app//com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:180)
        at java.base@18/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539)
        at java.base@18/java.util.concurrent.FutureTask.run(FutureTask.java:264)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$260/0x0000000800f6b520.run(Unknown Source)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  46) Thread[id=10753, name=httpShardExecutor-7313-thread-1, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@18/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$219/0x0000000800ec0530.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$260/0x0000000800f6b520.run(Unknown Source)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  47) Thread[id=10765, name=CloserThread, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/java.lang.Object.wait(Native Method)
        at java.base@18/java.lang.Object.wait(Object.java:338)
        at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2409)
  48) Thread[id=10800, name=qtp1740280584-10800, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@18/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$219/0x0000000800ec0530.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  49) Thread[id=10905, name=h2sc-7315-thread-3, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@18/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3464)
        at java.base@18/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3435)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  50) Thread[id=10865, name=searcherExecutor-7375-thread-1, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@18/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3464)
        at java.base@18/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3435)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at java.base@18/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  51) Thread[id=10903, name=HttpClient@7841103c-scheduler-1, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at java.base@18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  52) Thread[id=10773, name=qtp110215977-10773-acceptor-0@7fe48efd-ServerConnector@6480a6bb{HTTP/1.1, (http/1.1, h2c)}{127.0.0.1:53703}, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.Net.accept(Native Method)
        at java.base@18/sun.nio.ch.ServerSocketChannelImpl.implAccept(ServerSocketChannelImpl.java:425)
        at java.base@18/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:391)
        at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388)
        at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:702)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  53) Thread[id=10910, name=zkCallback-7357-thread-6, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  54) Thread[id=10774, name=qtp110215977-10774, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:361)
        at app//org.apache.solr.common.cloud.ZkStateReader.fetchCollectionProperties(ZkStateReader.java:1195)
        at app//org.apache.solr.common.cloud.ZkStateReader.getCollectionProperties(ZkStateReader.java:1142)
        at app//org.apache.solr.update.processor.RoutedAliasUpdateProcessor.wrap(RoutedAliasUpdateProcessor.java:104)
        at app//org.apache.solr.update.processor.DistributedUpdateProcessorFactory.getInstance(DistributedUpdateProcessorFactory.java:62)
        at app//org.apache.solr.update.processor.UpdateRequestProcessorChain.createProcessor(UpdateRequestProcessorChain.java:243)
        at app//org.apache.solr.update.processor.UpdateRequestProcessorChain.createProcessor(UpdateRequestProcessorChain.java:215)
        at app//org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:70)
        at app//org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:207)
        at app//org.apache.solr.core.SolrCore.execute(SolrCore.java:2866)
        at app//org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:887)
        at app//org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:606)
        at app//org.apache.solr.servlet.SolrDispatchFilter.dispatch(SolrDispatchFilter.java:239)
        at app//org.apache.solr.servlet.SolrDispatchFilter.lambda$doFilter$0(SolrDispatchFilter.java:207)
        at app//org.apache.solr.servlet.SolrDispatchFilter$$Lambda$381/0x000000080103f990.run(Unknown Source)
        at app//org.apache.solr.servlet.ServletUtils.traceHttpRequestExecution2(ServletUtils.java:257)
        at app//org.apache.solr.servlet.ServletUtils.rateLimitRequest(ServletUtils.java:227)
        at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:202)
        at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:184)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
        at app//org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:187)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
        at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
        at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
        at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1434)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
        at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)
        at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
        at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1349)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
        at app//org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
        at app//org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:713)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
        at app//org.eclipse.jetty.server.Server.handle(Server.java:516)
        at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:400)
        at app//org.eclipse.jetty.server.HttpChannel$$Lambda$379/0x0000000801039a00.dispatch(Unknown Source)
        at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:645)
        at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:392)
        at app//org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:352)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  55) Thread[id=10771, name=qtp110215977-10771, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@18/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$219/0x0000000800ec0530.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  56) Thread[id=10801, name=qtp1740280584-10801-acceptor-0@5910fe5b-ServerConnector@7c1fbca7{HTTP/1.1, (http/1.1, h2c)}{127.0.0.1:53738}, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.Net.accept(Native Method)
        at java.base@18/sun.nio.ch.ServerSocketChannelImpl.implAccept(ServerSocketChannelImpl.java:425)
        at java.base@18/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:391)
        at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388)
        at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:702)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  57) Thread[id=10868, name=searcherExecutor-7381-thread-1, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@18/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3464)
        at java.base@18/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3435)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at java.base@18/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  58) Thread[id=10741, name=qtp1804511539-10741, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@18/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$219/0x0000000800ec0530.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  59) Thread[id=10838, name=Connector-Scheduler-6fb1309a-1, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at java.base@18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  60) Thread[id=10782, name=httpShardExecutor-7332-thread-1, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@18/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$219/0x0000000800ec0530.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$260/0x0000000800f6b520.run(Unknown Source)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  61) Thread[id=10776, name=qtp110215977-10776, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  62) Thread[id=10770, name=qtp110215977-10770, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@18/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$219/0x0000000800ec0530.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  63) Thread[id=10749, name=Session-HouseKeeper-3a06ffc4-1, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at java.base@18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  64) Thread[id=10873, name=SocketProxy-Response-53738:53806, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.Net.poll(Native Method)
        at java.base@18/sun.nio.ch.NioSocketImpl.park(NioSocketImpl.java:178)
        at java.base@18/sun.nio.ch.NioSocketImpl.timedRead(NioSocketImpl.java:282)
        at java.base@18/sun.nio.ch.NioSocketImpl.implRead(NioSocketImpl.java:306)
        at java.base@18/sun.nio.ch.NioSocketImpl.read(NioSocketImpl.java:347)
        at java.base@18/sun.nio.ch.NioSocketImpl$1.read(NioSocketImpl.java:800)
        at java.base@18/java.net.Socket$SocketInputStream.read(Socket.java:966)
        at java.base@18/java.io.InputStream.read(InputStream.java:218)
        at app//org.apache.solr.client.solrj.cloud.SocketProxy$Bridge$Pump.run(SocketProxy.java:375)
  65) Thread[id=10911, name=zkCallback-7357-thread-7, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  66) Thread[id=10742, name=qtp1804511539-10742, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@18/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$219/0x0000000800ec0530.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  67) Thread[id=10778, name=Session-HouseKeeper-377197e3-1, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at java.base@18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  68) Thread[id=10870, name=HttpClient@7dc56056-scheduler-1, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at java.base@18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  69) Thread[id=10840, name=watches-7321-thread-1, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  70) Thread[id=10793, name=CloserThread, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/java.lang.Object.wait(Native Method)
        at java.base@18/java.lang.Object.wait(Object.java:338)
        at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2409)
  71) Thread[id=10904, name=h2sc-7315-thread-2, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@18/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3464)
        at java.base@18/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3435)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  72) Thread[id=10785, name=h2sc-7334-thread-1, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@18/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$219/0x0000000800ec0530.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$260/0x0000000800f6b520.run(Unknown Source)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  73) Thread[id=10823, name=SocketProxy-Acceptor-53737, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.Net.poll(Native Method)
        at java.base@18/sun.nio.ch.NioSocketImpl.park(NioSocketImpl.java:178)
        at java.base@18/sun.nio.ch.NioSocketImpl.timedAccept(NioSocketImpl.java:707)
        at java.base@18/sun.nio.ch.NioSocketImpl.accept(NioSocketImpl.java:749)
        at java.base@18/java.net.ServerSocket.implAccept(ServerSocket.java:673)
        at java.base@18/java.net.ServerSocket.platformImplAccept(ServerSocket.java:639)
        at java.base@18/java.net.ServerSocket.implAccept(ServerSocket.java:615)
        at java.base@18/java.net.ServerSocket.implAccept(ServerSocket.java:572)
        at java.base@18/java.net.ServerSocket.accept(ServerSocket.java:530)
        at app//org.apache.solr.client.solrj.cloud.SocketProxy$Acceptor.run(SocketProxy.java:445)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  74) Thread[id=10861, name=watches-7359-thread-1, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  75) Thread[id=10795, name=SocketProxy-Acceptor-53702, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.Net.poll(Native Method)
        at java.base@18/sun.nio.ch.NioSocketImpl.park(NioSocketImpl.java:178)
        at java.base@18/sun.nio.ch.NioSocketImpl.timedAccept(NioSocketImpl.java:707)
        at java.base@18/sun.nio.ch.NioSocketImpl.accept(NioSocketImpl.java:749)
        at java.base@18/java.net.ServerSocket.implAccept(ServerSocket.java:673)
        at java.base@18/java.net.ServerSocket.platformImplAccept(ServerSocket.java:639)
        at java.base@18/java.net.ServerSocket.implAccept(ServerSocket.java:615)
        at java.base@18/java.net.ServerSocket.implAccept(ServerSocket.java:572)
        at java.base@18/java.net.ServerSocket.accept(ServerSocket.java:530)
        at app//org.apache.solr.client.solrj.cloud.SocketProxy$Acceptor.run(SocketProxy.java:445)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  76) Thread[id=10802, name=qtp1740280584-10802, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  77) Thread[id=10916, name=zkCallback-7319-thread-4, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  78) Thread[id=10764, name=zkCallback-7319-thread-1, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:328)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.ensureExists(ZkCmdExecutor.java:145)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.ensureExists(ZkCmdExecutor.java:104)
        at app//org.apache.solr.cloud.DistributedMap.<init>(DistributedMap.java:47)
        at app//org.apache.solr.cloud.SizeLimitedDistributedMap.<init>(SizeLimitedDistributedMap.java:47)
        at app//org.apache.solr.cloud.Overseer.getFailureMap(Overseer.java:1062)
        at app//org.apache.solr.cloud.Overseer$ClusterStateUpdater.<init>(Overseer.java:207)
        at app//org.apache.solr.cloud.Overseer.start(Overseer.java:729)
        at app//org.apache.solr.cloud.OverseerElectionContext.runLeaderProcess(OverseerElectionContext.java:77)
        at app//org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:175)
        at app//org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135)
        at app//org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:355)
        at app//org.apache.solr.common.cloud.SolrZkClient$ProcessWatchWithExecutor.lambda$process$1(SolrZkClient.java:922)
        at app//org.apache.solr.common.cloud.SolrZkClient$ProcessWatchWithExecutor$$Lambda$490/0x00000008010d4000.run(Unknown Source)
        at java.base@18/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539)
        at java.base@18/java.util.concurrent.FutureTask.run(FutureTask.java:264)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$260/0x0000000800f6b520.run(Unknown Source)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  79) Thread[id=10803, name=qtp1740280584-10803, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  80) Thread[id=10890, name=watches-7340-thread-2-processing-127.0.0.1:53702_kfm collection1_shard1_replica_n2 collection1 shard1 core_node4, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkStateReader$CacheCleaner.run(ZkStateReader.java:2332)
        at java.base@18/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539)
        at java.base@18/java.util.concurrent.FutureTask.run(FutureTask.java:264)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$260/0x0000000800f6b520.run(Unknown Source)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  81) Thread[id=10775, name=qtp110215977-10775, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  82) Thread[id=10746, name=qtp1804511539-10746, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
	at __randomizedtesting.SeedInfo.seed([8D231CA2FCAC3F10]:0)


FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=10819, name=zkConnectionManagerCallback-7358-thread-1, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@18/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3464)
        at java.base@18/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3435)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at java.base@18/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
   2) Thread[id=10777, name=qtp110215977-10777, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
   3) Thread[id=10912, name=zkCallback-7357-thread-8, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
   4) Thread[id=10791, name=zkConnectionManagerCallback-7339-thread-1, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@18/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3464)
        at java.base@18/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3435)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at java.base@18/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
   5) Thread[id=10884, name=updateExecutor-7352-thread-1, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
   6) Thread[id=10846, name=searcherExecutor-7366-thread-1, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@18/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3464)
        at java.base@18/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3435)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at java.base@18/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
   7) Thread[id=10863, name=zkCallback-7357-thread-3, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
   8) Thread[id=10744, name=qtp1804511539-10744, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
   9) Thread[id=10804, name=qtp1740280584-10804, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  10) Thread[id=10869, name=zkCallback-7338-thread-3, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  11) Thread[id=10748, name=qtp1804511539-10748, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  12) Thread[id=10859, name=Connector-Scheduler-7c1fbca7-1, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at java.base@18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  13) Thread[id=10855, name=watches-7340-thread-1, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  14) Thread[id=10844, name=Connector-Scheduler-6480a6bb-1, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@18/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3464)
        at java.base@18/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3435)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at java.base@18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
        at java.base@18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  15) Thread[id=10792, name=zkCallback-7338-thread-1, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  16) Thread[id=10762, name=zkConnectionManagerCallback-7320-thread-1, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@18/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3464)
        at java.base@18/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3435)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at java.base@18/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  17) Thread[id=10813, name=h2sc-7353-thread-1, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@18/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$219/0x0000000800ec0530.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$260/0x0000000800f6b520.run(Unknown Source)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  18) Thread[id=10799, name=qtp1740280584-10799, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@18/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$219/0x0000000800ec0530.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  19) Thread[id=10747, name=qtp1804511539-10747, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  20) Thread[id=10862, name=zkCallback-7357-thread-2, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  21) Thread[id=10875, name=watches-7321-thread-2, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  22) Thread[id=10743, name=qtp1804511539-10743, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@18/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$219/0x0000000800ec0530.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  23) Thread[id=10867, name=zkCallback-7338-thread-2, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  24) Thread[id=10756, name=h2sc-7315-thread-1, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@18/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$219/0x0000000800ec0530.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$260/0x0000000800f6b520.run(Unknown Source)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  25) Thread[id=10902, name=updateExecutor-7314-thread-1, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  26) Thread[id=10913, name=zkCallback-7319-thread-3, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  27) Thread[id=10798, name=qtp1740280584-10798, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@18/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$219/0x0000000800ec0530.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  28) Thread[id=10772, name=qtp110215977-10772, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@18/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$219/0x0000000800ec0530.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  29) Thread[id=10810, name=httpShardExecutor-7351-thread-1, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@18/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$219/0x0000000800ec0530.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$260/0x0000000800f6b520.run(Unknown Source)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  30) Thread[id=10915, name=zkCallback-7338-thread-5, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  31) Thread[id=10745, name=qtp1804511539-10745, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  32) Thread[id=10908, name=zkCallback-7357-thread-4, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  33) Thread[id=10914, name=zkCallback-7338-thread-4, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  34) Thread[id=10821, name=CloserThread, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/java.lang.Object.wait(Native Method)
        at java.base@18/java.lang.Object.wait(Object.java:338)
        at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2409)
  35) Thread[id=10909, name=zkCallback-7357-thread-5, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  36) Thread[id=10805, name=qtp1740280584-10805, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  37) Thread[id=10874, name=zkCallback-7319-thread-2, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  38) Thread[id=10806, name=Session-HouseKeeper-5502908a-1, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at java.base@18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  39) Thread[id=10885, name=recoveryExecutor-7354-thread-1, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  40) Thread[id=10753, name=httpShardExecutor-7313-thread-1, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@18/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$219/0x0000000800ec0530.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$260/0x0000000800f6b520.run(Unknown Source)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  41) Thread[id=10765, name=CloserThread, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/java.lang.Object.wait(Native Method)
        at java.base@18/java.lang.Object.wait(Object.java:338)
        at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2409)
  42) Thread[id=10800, name=qtp1740280584-10800, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@18/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$219/0x0000000800ec0530.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  43) Thread[id=10905, name=h2sc-7315-thread-3, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@18/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3464)
        at java.base@18/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3435)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  44) Thread[id=10865, name=searcherExecutor-7375-thread-1, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@18/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3464)
        at java.base@18/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3435)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at java.base@18/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  45) Thread[id=10903, name=HttpClient@7841103c-scheduler-1, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at java.base@18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  46) Thread[id=10773, name=qtp110215977-10773, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  47) Thread[id=10910, name=zkCallback-7357-thread-6, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  48) Thread[id=10774, name=qtp110215977-10774, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  49) Thread[id=10771, name=qtp110215977-10771, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@18/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$219/0x0000000800ec0530.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  50) Thread[id=10801, name=qtp1740280584-10801, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  51) Thread[id=10868, name=searcherExecutor-7381-thread-1, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@18/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3464)
        at java.base@18/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3435)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at java.base@18/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  52) Thread[id=10741, name=qtp1804511539-10741, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@18/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$219/0x0000000800ec0530.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  53) Thread[id=10838, name=Connector-Scheduler-6fb1309a-1, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@18/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3464)
        at java.base@18/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3435)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at java.base@18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
        at java.base@18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  54) Thread[id=10782, name=httpShardExecutor-7332-thread-1, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@18/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$219/0x0000000800ec0530.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$260/0x0000000800f6b520.run(Unknown Source)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  55) Thread[id=10776, name=qtp110215977-10776, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  56) Thread[id=10770, name=qtp110215977-10770, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@18/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$219/0x0000000800ec0530.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  57) Thread[id=10749, name=Session-HouseKeeper-3a06ffc4-1, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at java.base@18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  58) Thread[id=10911, name=zkCallback-7357-thread-7, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  59) Thread[id=10742, name=qtp1804511539-10742, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@18/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$219/0x0000000800ec0530.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  60) Thread[id=10778, name=Session-HouseKeeper-377197e3-1, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at java.base@18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  61) Thread[id=10870, name=HttpClient@7dc56056-scheduler-1, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@18/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3464)
        at java.base@18/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3435)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at java.base@18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
        at java.base@18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  62) Thread[id=10840, name=watches-7321-thread-1, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  63) Thread[id=10793, name=CloserThread, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/java.lang.Object.wait(Native Method)
        at java.base@18/java.lang.Object.wait(Object.java:338)
        at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2409)
  64) Thread[id=10904, name=h2sc-7315-thread-2, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@18/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3464)
        at java.base@18/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3435)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  65) Thread[id=10785, name=h2sc-7334-thread-1, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@18/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$219/0x0000000800ec0530.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$260/0x0000000800f6b520.run(Unknown Source)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  66) Thread[id=10861, name=watches-7359-thread-1, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  67) Thread[id=10802, name=qtp1740280584-10802, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  68) Thread[id=10916, name=zkCallback-7319-thread-4, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  69) Thread[id=10764, name=zkCallback-7319-thread-1, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  70) Thread[id=10803, name=qtp1740280584-10803, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  71) Thread[id=10890, name=watches-7340-thread-2, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  72) Thread[id=10775, name=qtp110215977-10775, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  73) Thread[id=10746, name=qtp1804511539-10746, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@18/java.lang.Thread.run(Thread.java:833)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=10819, name=zkConnectionManagerCallback-7358-thread-1, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@18/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3464)
        at java.base@18/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3435)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at java.base@18/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
   2) Thread[id=10777, name=qtp110215977-10777, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
   3) Thread[id=10912, name=zkCallback-7357-thread-8, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
   4) Thread[id=10791, name=zkConnectionManagerCallback-7339-thread-1, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@18/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3464)
        at java.base@18/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3435)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at java.base@18/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
   5) Thread[id=10884, name=updateExecutor-7352-thread-1, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
   6) Thread[id=10846, name=searcherExecutor-7366-thread-1, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@18/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3464)
        at java.base@18/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3435)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at java.base@18/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
   7) Thread[id=10863, name=zkCallback-7357-thread-3, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
   8) Thread[id=10744, name=qtp1804511539-10744, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
   9) Thread[id=10804, name=qtp1740280584-10804, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  10) Thread[id=10869, name=zkCallback-7338-thread-3, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  11) Thread[id=10748, name=qtp1804511539-10748, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  12) Thread[id=10859, name=Connector-Scheduler-7c1fbca7-1, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at java.base@18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  13) Thread[id=10855, name=watches-7340-thread-1, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  14) Thread[id=10844, name=Connector-Scheduler-6480a6bb-1, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@18/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3464)
        at java.base@18/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3435)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at java.base@18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
        at java.base@18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  15) Thread[id=10792, name=zkCallback-7338-thread-1, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  16) Thread[id=10762, name=zkConnectionManagerCallback-7320-thread-1, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@18/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3464)
        at java.base@18/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3435)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at java.base@18/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  17) Thread[id=10813, name=h2sc-7353-thread-1, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@18/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$219/0x0000000800ec0530.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$260/0x0000000800f6b520.run(Unknown Source)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  18) Thread[id=10799, name=qtp1740280584-10799, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@18/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$219/0x0000000800ec0530.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  19) Thread[id=10747, name=qtp1804511539-10747, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  20) Thread[id=10862, name=zkCallback-7357-thread-2, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  21) Thread[id=10875, name=watches-7321-thread-2, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  22) Thread[id=10743, name=qtp1804511539-10743, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@18/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$219/0x0000000800ec0530.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  23) Thread[id=10867, name=zkCallback-7338-thread-2, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  24) Thread[id=10756, name=h2sc-7315-thread-1, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@18/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$219/0x0000000800ec0530.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$260/0x0000000800f6b520.run(Unknown Source)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  25) Thread[id=10902, name=updateExecutor-7314-thread-1, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  26) Thread[id=10913, name=zkCallback-7319-thread-3, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  27) Thread[id=10798, name=qtp1740280584-10798, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@18/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$219/0x0000000800ec0530.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  28) Thread[id=10772, name=qtp110215977-10772, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@18/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$219/0x0000000800ec0530.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  29) Thread[id=10810, name=httpShardExecutor-7351-thread-1, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@18/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$219/0x0000000800ec0530.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$260/0x0000000800f6b520.run(Unknown Source)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  30) Thread[id=10915, name=zkCallback-7338-thread-5, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  31) Thread[id=10745, name=qtp1804511539-10745, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  32) Thread[id=10908, name=zkCallback-7357-thread-4, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  33) Thread[id=10914, name=zkCallback-7338-thread-4, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  34) Thread[id=10821, name=CloserThread, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/java.lang.Object.wait(Native Method)
        at java.base@18/java.lang.Object.wait(Object.java:338)
        at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2409)
  35) Thread[id=10909, name=zkCallback-7357-thread-5, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  36) Thread[id=10805, name=qtp1740280584-10805, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  37) Thread[id=10874, name=zkCallback-7319-thread-2, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  38) Thread[id=10806, name=Session-HouseKeeper-5502908a-1, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at java.base@18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  39) Thread[id=10885, name=recoveryExecutor-7354-thread-1, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  40) Thread[id=10753, name=httpShardExecutor-7313-thread-1, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@18/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$219/0x0000000800ec0530.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$260/0x0000000800f6b520.run(Unknown Source)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  41) Thread[id=10765, name=CloserThread, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/java.lang.Object.wait(Native Method)
        at java.base@18/java.lang.Object.wait(Object.java:338)
        at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2409)
  42) Thread[id=10800, name=qtp1740280584-10800, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@18/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$219/0x0000000800ec0530.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  43) Thread[id=10905, name=h2sc-7315-thread-3, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@18/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3464)
        at java.base@18/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3435)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  44) Thread[id=10865, name=searcherExecutor-7375-thread-1, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@18/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3464)
        at java.base@18/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3435)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at java.base@18/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  45) Thread[id=10903, name=HttpClient@7841103c-scheduler-1, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at java.base@18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  46) Thread[id=10773, name=qtp110215977-10773, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  47) Thread[id=10910, name=zkCallback-7357-thread-6, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  48) Thread[id=10774, name=qtp110215977-10774, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  49) Thread[id=10771, name=qtp110215977-10771, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@18/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$219/0x0000000800ec0530.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  50) Thread[id=10801, name=qtp1740280584-10801, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  51) Thread[id=10868, name=searcherExecutor-7381-thread-1, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@18/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3464)
        at java.base@18/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3435)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at java.base@18/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  52) Thread[id=10741, name=qtp1804511539-10741, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@18/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$219/0x0000000800ec0530.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  53) Thread[id=10838, name=Connector-Scheduler-6fb1309a-1, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@18/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3464)
        at java.base@18/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3435)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at java.base@18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
        at java.base@18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  54) Thread[id=10782, name=httpShardExecutor-7332-thread-1, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@18/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$219/0x0000000800ec0530.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$260/0x0000000800f6b520.run(Unknown Source)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  55) Thread[id=10776, name=qtp110215977-10776, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  56) Thread[id=10770, name=qtp110215977-10770, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@18/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$219/0x0000000800ec0530.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  57) Thread[id=10749, name=Session-HouseKeeper-3a06ffc4-1, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at java.base@18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  58) Thread[id=10911, name=zkCallback-7357-thread-7, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  59) Thread[id=10742, name=qtp1804511539-10742, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@18/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$219/0x0000000800ec0530.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  60) Thread[id=10778, name=Session-HouseKeeper-377197e3-1, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at java.base@18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  61) Thread[id=10870, name=HttpClient@7dc56056-scheduler-1, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@18/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3464)
        at java.base@18/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3435)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at java.base@18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
        at java.base@18/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  62) Thread[id=10840, name=watches-7321-thread-1, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  63) Thread[id=10793, name=CloserThread, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/java.lang.Object.wait(Native Method)
        at java.base@18/java.lang.Object.wait(Object.java:338)
        at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2409)
  64) Thread[id=10904, name=h2sc-7315-thread-2, state=WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@18/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3464)
        at java.base@18/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3435)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  65) Thread[id=10785, name=h2sc-7334-thread-1, state=RUNNABLE, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@18/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@18/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@18/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$219/0x0000000800ec0530.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$260/0x0000000800f6b520.run(Unknown Source)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  66) Thread[id=10861, name=watches-7359-thread-1, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  67) Thread[id=10802, name=qtp1740280584-10802, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  68) Thread[id=10916, name=zkCallback-7319-thread-4, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  69) Thread[id=10764, name=zkCallback-7319-thread-1, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  70) Thread[id=10803, name=qtp1740280584-10803, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  71) Thread[id=10890, name=watches-7340-thread-2, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@18/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@18/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  72) Thread[id=10775, name=qtp110215977-10775, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
  73) Thread[id=10746, name=qtp1804511539-10746, state=TIMED_WAITING, group=TGRP-LeaderFailoverAfterPartitionTest]
        at java.base@18/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@18/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@18/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
	at __randomizedtesting.SeedInfo.seed([8D231CA2FCAC3F10]:0)


FAILED:  org.apache.solr.cloud.TestStressLiveNodes.testStress

Error Message:
java.lang.AssertionError: iter5 expected:<13> but was:<16>

Stack Trace:
java.lang.AssertionError: iter5 expected:<13> but was:<16>
	at __randomizedtesting.SeedInfo.seed([8D231CA2FCAC3F10:983861637F5BC76B]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.junit.Assert.failNotEquals(Assert.java:835)
	at org.junit.Assert.assertEquals(Assert.java:647)
	at org.apache.solr.cloud.TestStressLiveNodes.testStress(TestStressLiveNodes.java:211)
	at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
	at java.base/java.lang.reflect.Method.invoke(Method.java:577)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base/java.lang.Thread.run(Thread.java:833)


FAILED:  org.apache.solr.cloud.HttpPartitionOnCommitTest.test

Error Message:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:54281/rt: Underlying core creation failed while creating collection: c8n_1x3_commits

Stack Trace:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:54281/rt: Underlying core creation failed while creating collection: c8n_1x3_commits
	at __randomizedtesting.SeedInfo.seed([8D231CA2FCAC3F10:5772378525052E8]:0)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:715)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at app//org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1193)
	at app//org.apache.solr.cloud.AbstractBasicDistributedZkTestBase.createCollection(AbstractBasicDistributedZkTestBase.java:1177)
	at app//org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:2025)
	at app//org.apache.solr.cloud.HttpPartitionOnCommitTest.oneShardTest(HttpPartitionOnCommitTest.java:132)
	at app//org.apache.solr.cloud.HttpPartitionOnCommitTest.test(HttpPartitionOnCommitTest.java:68)
	at java.base@18/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
	at java.base@18/java.lang.reflect.Method.invoke(Method.java:577)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1164)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1135)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base@18/java.lang.Thread.run(Thread.java:833)


FAILED:  org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=1079, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@18/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:361)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:75)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getLines(AbstractAnalysisFactory.java:302)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getWordSet(AbstractAnalysisFactory.java:293)
        at app//org.apache.lucene.analysis.en.AbstractWordsFileFilterFactory.inform(AbstractWordsFileFilterFactory.java:88)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:762)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1470)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1319)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1307)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:654)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:188)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:119)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:279)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:345)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:256)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$632/0x0000000801135978.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:49)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:252)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1850)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3394)
        at app//org.apache.solr.core.SolrCore$$Lambda$743/0x00000008011f3320.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2761)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1153/0x00000008014e2a40.run(Unknown Source)
        at java.base@18/java.lang.Thread.run(Thread.java:833)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=1079, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@18/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:361)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:75)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getLines(AbstractAnalysisFactory.java:302)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getWordSet(AbstractAnalysisFactory.java:293)
        at app//org.apache.lucene.analysis.en.AbstractWordsFileFilterFactory.inform(AbstractWordsFileFilterFactory.java:88)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:762)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1470)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1319)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1307)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:654)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:188)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:119)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:279)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:345)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:256)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$632/0x0000000801135978.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:49)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:252)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1850)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3394)
        at app//org.apache.solr.core.SolrCore$$Lambda$743/0x00000008011f3320.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2761)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1153/0x00000008014e2a40.run(Unknown Source)
        at java.base@18/java.lang.Thread.run(Thread.java:833)
	at __randomizedtesting.SeedInfo.seed([8D231CA2FCAC3F10]:0)

[JENKINS] Solr-main-MacOSX (64bit/jdk-17.0.2) - Build # 868 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/868/
Java: 64bit/jdk-17.0.2 -XX:-UseCompressedOops -XX:+UseShenandoahGC

3 tests failed.
FAILED:  org.apache.solr.cloud.HttpPartitionOnCommitTest.test

Error Message:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:56957: Underlying core creation failed while creating collection: control_collection

Stack Trace:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:56957: Underlying core creation failed while creating collection: control_collection
	at __randomizedtesting.SeedInfo.seed([E50B767A00002F4A:6D5F49A0AEFC42B2]:0)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:715)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:405)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:371)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1248)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:969)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:896)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:223)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:238)
	at app//org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:357)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1162)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1135)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)


FAILED:  org.apache.solr.cloud.TestRandomRequestDistribution.test

Error Message:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:56203/ex: Underlying core creation failed while creating collection: a1x2

Stack Trace:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:56203/ex: Underlying core creation failed while creating collection: a1x2
	at __randomizedtesting.SeedInfo.seed([E50B767A00002F4A:6D5F49A0AEFC42B2]:0)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:715)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:405)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:371)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1248)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:969)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:896)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:223)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:238)
	at app//org.apache.solr.cloud.TestRandomRequestDistribution.testRequestTracking(TestRandomRequestDistribution.java:76)
	at app//org.apache.solr.cloud.TestRandomRequestDistribution.test(TestRandomRequestDistribution.java:67)
	at java.base@17.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base@17.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base@17.0.2/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base@17.0.2/java.lang.reflect.Method.invoke(Method.java:568)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1164)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1135)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)


FAILED:  org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=1056, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@17.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:328)
        at app//org.apache.solr.cloud.ZkController.pathExists(ZkController.java:1228)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:73)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getSnowballWordSet(AbstractAnalysisFactory.java:321)
        at app//org.apache.lucene.analysis.en.AbstractWordsFileFilterFactory.inform(AbstractWordsFileFilterFactory.java:90)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:762)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1470)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1319)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1307)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:654)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:188)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:119)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:279)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:345)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:256)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$633/0x00000008011534f0.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:49)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:252)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1850)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3394)
        at app//org.apache.solr.core.SolrCore$$Lambda$743/0x00000008012157d8.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2761)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1186/0x0000000801650678.run(Unknown Source)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=1056, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@17.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:328)
        at app//org.apache.solr.cloud.ZkController.pathExists(ZkController.java:1228)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:73)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getSnowballWordSet(AbstractAnalysisFactory.java:321)
        at app//org.apache.lucene.analysis.en.AbstractWordsFileFilterFactory.inform(AbstractWordsFileFilterFactory.java:90)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:762)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1470)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1319)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1307)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:654)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:188)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:119)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:279)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:345)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:256)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$633/0x00000008011534f0.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:49)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:252)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1850)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3394)
        at app//org.apache.solr.core.SolrCore$$Lambda$743/0x00000008012157d8.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2761)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1186/0x0000000801650678.run(Unknown Source)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
	at __randomizedtesting.SeedInfo.seed([E50B767A00002F4A]:0)

[JENKINS] Solr-main-MacOSX (64bit/jdk-16.0.2) - Build # 867 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/867/
Java: 64bit/jdk-16.0.2 -XX:+UseCompressedOops -XX:+UseSerialGC

4 tests failed.
FAILED:  org.apache.solr.cloud.DocValuesNotIndexedTest.testGroupingDVOnlySortLast

Error Message:
java.lang.AssertionError: Unexpected number of elements in the group for 'intGSL' size: '9' GroupValue: 'null' rsp: {responseHeader={zkConnected=true, status=0, QTime=72, params={q=*:*, group.sort=id asc, _stateVer_=dv_coll:13, group.limit=100, rows=100, wt=javabin, version=2, group.field=intGSL, group=true}}, grouped={intGSL={matches=45, groups=[{groupValue=null, doclist={numFound=9,numFoundExact=true,start=0,maxScore=1.0,docs=[SolrDocument{id=10000, _version_=1729380957052469248, _root_=10000}, SolrDocument{id=10010, _version_=1729380957058760704, _root_=10010}, SolrDocument{id=10015, _version_=1729380957054566405, _root_=10015}, SolrDocument{id=10020, _version_=1729380957055614976, _root_=10020}, SolrDocument{id=10025, _version_=1729380957058760706, _root_=10025}, SolrDocument{id=10030, _version_=1729380957056663557, _root_=10030}, SolrDocument{id=10035, _version_=1729380957058760710, _root_=10035}, SolrDocument{id=10040, _version_=1729380957061906432, _root_=10040}, SolrDocument{id=10045, _version_=1729380957058760712, _root_=10045}]}}, {groupValue=1050845894, doclist={numFound=7,numFoundExact=true,start=0,maxScore=1.0,docs=[SolrDocument{id=0, intGSL=1050845894, longGSL=4557485844322906447, doubleGSL=8935.060006510374, floatGSL=5951.12, dateGSL=Sun May 22 15:01:47 GMT+14:00 89431712, stringGSL=base_string_815184__00008293, boolGSL=true, sortableGSL=base_string_337557__00006205, _version_=1729380957047226368, _root_=0}, SolrDocument{id=1, intGSL=1050845894, longGSL=4557485844322906447, doubleGSL=8935.060006510374, floatGSL=5951.12, dateGSL=Sun May 22 15:01:47 GMT+14:00 89431712, stringGSL=base_string_815184__00008293, boolGSL=false, sortableGSL=base_string_337557__00006205, _version_=1729380957048274944, _root_=1}, SolrDocument{id=2, intGSL=1050845894, longGSL=4557485844322906447, doubleGSL=8935.060006510374, floatGSL=5951.12, dateGSL=Sun May 22 15:01:47 GMT+14:00 89431712, stringGSL=base_string_815184__00008293, boolGSL=true, sortableGSL=base_string_337557__00006205, _version_=1729380957051420672, _root_=2}, SolrDocument{id=3, intGSL=1050845894, longGSL=4557485844322906447, doubleGSL=8935.060006510374, floatGSL=5951.12, dateGSL=Sun May 22 15:01:47 GMT+14:00 89431712, stringGSL=base_string_815184__00008293, boolGSL=false, sortableGSL=base_string_337557__00006205, _version_=1729380957057712128, _root_=3}, SolrDocument{id=4, intGSL=1050845894, longGSL=4557485844322906447, doubleGSL=8935.060006510374, floatGSL=5951.12, dateGSL=Sun May 22 15:01:47 GMT+14:00 89431712, stringGSL=base_string_815184__00008293, boolGSL=true, sortableGSL=base_string_337557__00006205, _version_=1729380957060857856, _root_=4}, SolrDocument{id=5, intGSL=1050845894, longGSL=4557485844322906447, doubleGSL=8935.060006510374, floatGSL=5951.12, dateGSL=Sun May 22 15:01:47 GMT+14:00 89431712, stringGSL=base_string_815184__00008293, boolGSL=false, sortableGSL=base_string_337557__00006205, _version_=1729380957055614976, _root_=5}, SolrDocument{id=6, intGSL=1050845894, longGSL=4557485844322906447, doubleGSL=8935.060006510374, floatGSL=5951.12, dateGSL=Sun May 22 15:01:47 GMT+14:00 89431712, stringGSL=base_string_815184__00008293, boolGSL=true, sortableGSL=base_string_337557__00006205, _version_=1729380957057712129, _root_=6}]}}, {groupValue=1050848255, doclist={numFound=4,numFoundExact=true,start=0,maxScore=1.0,docs=[SolrDocument{id=14, intGSL=1050848255, longGSL=4557485844322920769, doubleGSL=20536.060006510375, floatGSL=20811.121, dateGSL=Sun May 22 15:01:53 GMT+14:00 89431712, stringGSL=base_string_815184__00012451, boolGSL=true, sortableGSL=base_string_337557__00014089, _version_=1729380957054566404, _root_=14}, SolrDocument{id=17, intGSL=1050848255, longGSL=4557485844322920769, doubleGSL=20536.060006510375, floatGSL=20811.121, dateGSL=Sun May 22 15:01:53 GMT+14:00 89431712, stringGSL=base_string_815184__00012451, boolGSL=false, sortableGSL=base_string_337557__00014089, _version_=1729380957055614977, _root_=17}, SolrDocument{id=18, intGSL=1050848255, longGSL=4557485844322920769, doubleGSL=20536.060006510375, floatGSL=20811.121, dateGSL=Sun May 22 15:01:53 GMT+14:00 89431712, stringGSL=base_string_815184__00012451, boolGSL=true, sortableGSL=base_string_337557__00014089, _version_=1729380957058760705, _root_=18}, SolrDocument{id=19, intGSL=1050848255, longGSL=4557485844322920769, doubleGSL=20536.060006510375, floatGSL=20811.121, dateGSL=Sun May 22 15:01:53 GMT+14:00 89431712, stringGSL=base_string_815184__00012451, boolGSL=false, sortableGSL=base_string_337557__00014089, _version_=1729380957056663552, _root_=19}]}}, {groupValue=1050849974, doclist={numFound=6,numFoundExact=true,start=0,maxScore=1.0,docs=[SolrDocument{id=21, intGSL=1050849974, longGSL=4557485844322923262, doubleGSL=23130.060006510375, floatGSL=23433.121, dateGSL=Sun May 22 15:01:58 GMT+14:00 89431712, stringGSL=base_string_815184__00020303, boolGSL=false, sortableGSL=base_string_337557__00021495, _version_=1729380957056663553, _root_=21}, SolrDocument{id=23, intGSL=1050849974, longGSL=4557485844322923262, doubleGSL=23130.060006510375, floatGSL=23433.121, dateGSL=Sun May 22 15:01:58 GMT+14:00 89431712, stringGSL=base_string_815184__00020303, boolGSL=false, sortableGSL=base_string_337557__00021495, _version_=1729380957056663554, _root_=23}, SolrDocument{id=24, intGSL=1050849974, longGSL=4557485844322923262, doubleGSL=23130.060006510375, floatGSL=23433.121, dateGSL=Sun May 22 15:01:58 GMT+14:00 89431712, stringGSL=base_string_815184__00020303, boolGSL=true, sortableGSL=base_string_337557__00021495, _version_=1729380957055614977, _root_=24}, SolrDocument{id=25, intGSL=1050849974, longGSL=4557485844322923262, doubleGSL=23130.060006510375, floatGSL=23433.121, dateGSL=Sun May 22 15:01:58 GMT+14:00 89431712, stringGSL=base_string_815184__00020303, boolGSL=false, sortableGSL=base_string_337557__00021495, _version_=1729380957056663555, _root_=25}, SolrDocument{id=26, intGSL=1050849974, longGSL=4557485844322923262, doubleGSL=23130.060006510375, floatGSL=23433.121, dateGSL=Sun May 22 15:01:58 GMT+14:00 89431712, stringGSL=base_string_815184__00020303, boolGSL=true, sortableGSL=base_string_337557__00021495, _version_=1729380957058760707, _root_=26}, SolrDocument{id=27, intGSL=1050849974, longGSL=4557485844322923262, doubleGSL=23130.060006510375, floatGSL=23433.121, dateGSL=Sun May 22 15:01:58 GMT+14:00 89431712, stringGSL=base_string_815184__00020303, boolGSL=false, sortableGSL=base_string_337557__00021495, _version_=1729380957055614978, _root_=27}]}}, {groupValue=1050858685, doclist={numFound=6,numFoundExact=true,start=0,maxScore=1.0,docs=[SolrDocument{id=28, intGSL=1050858685, longGSL=4557485844322925296, doubleGSL=26608.060006510375, floatGSL=26136.121, dateGSL=Sun May 22 15:02:03 GMT+14:00 89431712, stringGSL=base_string_815184__00022075, boolGSL=true, sortableGSL=base_string_337557__00024212, _version_=1729380957055614979, _root_=28}, SolrDocument{id=29, intGSL=1050858685, longGSL=4557485844322925296, doubleGSL=26608.060006510375, floatGSL=26136.121, dateGSL=Sun May 22 15:02:03 GMT+14:00 89431712, stringGSL=base_string_815184__00022075, boolGSL=false, sortableGSL=base_string_337557__00024212, _version_=1729380957056663556, _root_=29}, SolrDocument{id=30, intGSL=1050858685, longGSL=4557485844322925296, doubleGSL=26608.060006510375, floatGSL=26136.121, dateGSL=Sun May 22 15:02:03 GMT+14:00 89431712, stringGSL=base_string_815184__00022075, boolGSL=true, sortableGSL=base_string_337557__00024212, _version_=1729380957058760708, _root_=30}, SolrDocument{id=31, intGSL=1050858685, longGSL=4557485844322925296, doubleGSL=26608.060006510375, floatGSL=26136.121, dateGSL=Sun May 22 15:02:03 GMT+14:00 89431712, stringGSL=base_string_815184__00022075, boolGSL=false, sortableGSL=base_string_337557__00024212, _version_=1729380957058760709, _root_=31}, SolrDocument{id=32, intGSL=1050858685, longGSL=4557485844322925296, doubleGSL=26608.060006510375, floatGSL=26136.121, dateGSL=Sun May 22 15:02:03 GMT+14:00 89431712, stringGSL=base_string_815184__00022075, boolGSL=true, sortableGSL=base_string_337557__00024212, _version_=1729380957055614980, _root_=32}, SolrDocument{id=33, intGSL=1050858685, longGSL=4557485844322925296, doubleGSL=26608.060006510375, floatGSL=26136.121, dateGSL=Sun May 22 15:02:03 GMT+14:00 89431712, stringGSL=base_string_815184__00022075, boolGSL=false, sortableGSL=base_string_337557__00024212, _version_=1729380957056663558, _root_=33}]}}, {groupValue=1050860925, doclist={numFound=4,numFoundExact=true,start=0,maxScore=1.0,docs=[SolrDocument{id=37, intGSL=1050860925, longGSL=4557485844322927314, doubleGSL=30051.060006510375, floatGSL=26417.121, dateGSL=Sun May 22 15:02:09 GMT+14:00 89431712, stringGSL=base_string_815184__00030670, boolGSL=false, sortableGSL=base_string_337557__00031431, _version_=1729380957060857856, _root_=37}, SolrDocument{id=38, intGSL=1050860925, longGSL=4557485844322927314, doubleGSL=30051.060006510375, floatGSL=26417.121, dateGSL=Sun May 22 15:02:09 GMT+14:00 89431712, stringGSL=base_string_815184__00030670, boolGSL=true, sortableGSL=base_string_337557__00031431, _version_=1729380957055614981, _root_=38}, SolrDocument{id=39, intGSL=1050860925, longGSL=4557485844322927314, doubleGSL=30051.060006510375, floatGSL=26417.121, dateGSL=Sun May 22 15:02:09 GMT+14:00 89431712, stringGSL=base_string_815184__00030670, boolGSL=false, sortableGSL=base_string_337557__00031431, _version_=1729380957058760711, _root_=39}, SolrDocument{id=40, intGSL=1050860925, longGSL=4557485844322927314, doubleGSL=30051.060006510375, floatGSL=26417.121, dateGSL=Sun May 22 15:02:09 GMT+14:00 89431712, stringGSL=base_string_815184__00030670, boolGSL=true, sortableGSL=base_string_337557__00031431, _version_=1729380957056663552, _root_=40}]}}, {groupValue=1050867084, doclist={numFound=3,numFoundExact=true,start=0,maxScore=1.0,docs=[SolrDocument{id=42, intGSL=1050867084, longGSL=4557485844322937265, doubleGSL=37775.06000651037, floatGSL=29443.121, dateGSL=Sun May 22 15:02:10 GMT+14:00 89431712, stringGSL=base_string_815184__00032857, boolGSL=true, sortableGSL=base_string_337557__00033940, _version_=1729380957056663553, _root_=42}, SolrDocument{id=45, intGSL=1050867084, longGSL=4557485844322937265, doubleGSL=37775.06000651037, floatGSL=29443.121, dateGSL=Sun May 22 15:02:10 GMT+14:00 89431712, stringGSL=base_string_815184__00032857, boolGSL=false, sortableGSL=base_string_337557__00033940, _version_=1729380957061906433, _root_=45}, SolrDocument{id=48, intGSL=1050867084, longGSL=4557485844322937265, doubleGSL=37775.06000651037, floatGSL=29443.121, dateGSL=Sun May 22 15:02:10 GMT+14:00 89431712, stringGSL=base_string_815184__00032857, boolGSL=true, sortableGSL=base_string_337557__00033940, _version_=1729380957056663554, _root_=48}]}}, {groupValue=1050846662, doclist={numFound=6,numFoundExact=true,start=0,maxScore=1.0,docs=[SolrDocument{id=10, intGSL=1050846662, longGSL=4557485844322912948, doubleGSL=17896.060006510375, floatGSL=14448.12, dateGSL=Sun May 22 15:01:50 GMT+14:00 89431712, stringGSL=base_string_815184__00011699, boolGSL=true, sortableGSL=base_string_337557__00010270, _version_=1729380957054566401, _root_=10}, SolrDocument{id=11, intGSL=1050846662, longGSL=4557485844322912948, doubleGSL=17896.060006510375, floatGSL=14448.12, dateGSL=Sun May 22 15:01:50 GMT+14:00 89431712, stringGSL=base_string_815184__00011699, boolGSL=false, sortableGSL=base_string_337557__00010270, _version_=1729380957054566402, _root_=11}, SolrDocument{id=13, intGSL=1050846662, longGSL=4557485844322912948, doubleGSL=17896.060006510375, floatGSL=14448.12, dateGSL=Sun May 22 15:01:50 GMT+14:00 89431712, stringGSL=base_string_815184__00011699, boolGSL=false, sortableGSL=base_string_337557__00010270, _version_=1729380957054566403, _root_=13}, SolrDocument{id=7, intGSL=1050846662, longGSL=4557485844322912948, doubleGSL=17896.060006510375, floatGSL=14448.12, dateGSL=Sun May 22 15:01:50 GMT+14:00 89431712, stringGSL=base_string_815184__00011699, boolGSL=false, sortableGSL=base_string_337557__00010270, _version_=1729380957057712130, _root_=7}, SolrDocument{id=8, intGSL=1050846662, longGSL=4557485844322912948, doubleGSL=17896.060006510375, floatGSL=14448.12, dateGSL=Sun May 22 15:01:50 GMT+14:00 89431712, stringGSL=base_string_815184__00011699, boolGSL=true, sortableGSL=base_string_337557__00010270, _version_=1729380957054566400, _root_=8}, SolrDocument{id=9, intGSL=1050846662, longGSL=4557485844322912948, doubleGSL=17896.060006510375, floatGSL=14448.12, dateGSL=Sun May 22 15:01:50 GMT+14:00 89431712, stringGSL=base_string_815184__00011699, boolGSL=false, sortableGSL=base_string_337557__00010270, _version_=1729380957057712131, _root_=9}]}}]}}}

Stack Trace:
java.lang.AssertionError: Unexpected number of elements in the group for 'intGSL' size: '9' GroupValue: 'null' rsp: {responseHeader={zkConnected=true, status=0, QTime=72, params={q=*:*, group.sort=id asc, _stateVer_=dv_coll:13, group.limit=100, rows=100, wt=javabin, version=2, group.field=intGSL, group=true}}, grouped={intGSL={matches=45, groups=[{groupValue=null, doclist={numFound=9,numFoundExact=true,start=0,maxScore=1.0,docs=[SolrDocument{id=10000, _version_=1729380957052469248, _root_=10000}, SolrDocument{id=10010, _version_=1729380957058760704, _root_=10010}, SolrDocument{id=10015, _version_=1729380957054566405, _root_=10015}, SolrDocument{id=10020, _version_=1729380957055614976, _root_=10020}, SolrDocument{id=10025, _version_=1729380957058760706, _root_=10025}, SolrDocument{id=10030, _version_=1729380957056663557, _root_=10030}, SolrDocument{id=10035, _version_=1729380957058760710, _root_=10035}, SolrDocument{id=10040, _version_=1729380957061906432, _root_=10040}, SolrDocument{id=10045, _version_=1729380957058760712, _root_=10045}]}}, {groupValue=1050845894, doclist={numFound=7,numFoundExact=true,start=0,maxScore=1.0,docs=[SolrDocument{id=0, intGSL=1050845894, longGSL=4557485844322906447, doubleGSL=8935.060006510374, floatGSL=5951.12, dateGSL=Sun May 22 15:01:47 GMT+14:00 89431712, stringGSL=base_string_815184__00008293, boolGSL=true, sortableGSL=base_string_337557__00006205, _version_=1729380957047226368, _root_=0}, SolrDocument{id=1, intGSL=1050845894, longGSL=4557485844322906447, doubleGSL=8935.060006510374, floatGSL=5951.12, dateGSL=Sun May 22 15:01:47 GMT+14:00 89431712, stringGSL=base_string_815184__00008293, boolGSL=false, sortableGSL=base_string_337557__00006205, _version_=1729380957048274944, _root_=1}, SolrDocument{id=2, intGSL=1050845894, longGSL=4557485844322906447, doubleGSL=8935.060006510374, floatGSL=5951.12, dateGSL=Sun May 22 15:01:47 GMT+14:00 89431712, stringGSL=base_string_815184__00008293, boolGSL=true, sortableGSL=base_string_337557__00006205, _version_=1729380957051420672, _root_=2}, SolrDocument{id=3, intGSL=1050845894, longGSL=4557485844322906447, doubleGSL=8935.060006510374, floatGSL=5951.12, dateGSL=Sun May 22 15:01:47 GMT+14:00 89431712, stringGSL=base_string_815184__00008293, boolGSL=false, sortableGSL=base_string_337557__00006205, _version_=1729380957057712128, _root_=3}, SolrDocument{id=4, intGSL=1050845894, longGSL=4557485844322906447, doubleGSL=8935.060006510374, floatGSL=5951.12, dateGSL=Sun May 22 15:01:47 GMT+14:00 89431712, stringGSL=base_string_815184__00008293, boolGSL=true, sortableGSL=base_string_337557__00006205, _version_=1729380957060857856, _root_=4}, SolrDocument{id=5, intGSL=1050845894, longGSL=4557485844322906447, doubleGSL=8935.060006510374, floatGSL=5951.12, dateGSL=Sun May 22 15:01:47 GMT+14:00 89431712, stringGSL=base_string_815184__00008293, boolGSL=false, sortableGSL=base_string_337557__00006205, _version_=1729380957055614976, _root_=5}, SolrDocument{id=6, intGSL=1050845894, longGSL=4557485844322906447, doubleGSL=8935.060006510374, floatGSL=5951.12, dateGSL=Sun May 22 15:01:47 GMT+14:00 89431712, stringGSL=base_string_815184__00008293, boolGSL=true, sortableGSL=base_string_337557__00006205, _version_=1729380957057712129, _root_=6}]}}, {groupValue=1050848255, doclist={numFound=4,numFoundExact=true,start=0,maxScore=1.0,docs=[SolrDocument{id=14, intGSL=1050848255, longGSL=4557485844322920769, doubleGSL=20536.060006510375, floatGSL=20811.121, dateGSL=Sun May 22 15:01:53 GMT+14:00 89431712, stringGSL=base_string_815184__00012451, boolGSL=true, sortableGSL=base_string_337557__00014089, _version_=1729380957054566404, _root_=14}, SolrDocument{id=17, intGSL=1050848255, longGSL=4557485844322920769, doubleGSL=20536.060006510375, floatGSL=20811.121, dateGSL=Sun May 22 15:01:53 GMT+14:00 89431712, stringGSL=base_string_815184__00012451, boolGSL=false, sortableGSL=base_string_337557__00014089, _version_=1729380957055614977, _root_=17}, SolrDocument{id=18, intGSL=1050848255, longGSL=4557485844322920769, doubleGSL=20536.060006510375, floatGSL=20811.121, dateGSL=Sun May 22 15:01:53 GMT+14:00 89431712, stringGSL=base_string_815184__00012451, boolGSL=true, sortableGSL=base_string_337557__00014089, _version_=1729380957058760705, _root_=18}, SolrDocument{id=19, intGSL=1050848255, longGSL=4557485844322920769, doubleGSL=20536.060006510375, floatGSL=20811.121, dateGSL=Sun May 22 15:01:53 GMT+14:00 89431712, stringGSL=base_string_815184__00012451, boolGSL=false, sortableGSL=base_string_337557__00014089, _version_=1729380957056663552, _root_=19}]}}, {groupValue=1050849974, doclist={numFound=6,numFoundExact=true,start=0,maxScore=1.0,docs=[SolrDocument{id=21, intGSL=1050849974, longGSL=4557485844322923262, doubleGSL=23130.060006510375, floatGSL=23433.121, dateGSL=Sun May 22 15:01:58 GMT+14:00 89431712, stringGSL=base_string_815184__00020303, boolGSL=false, sortableGSL=base_string_337557__00021495, _version_=1729380957056663553, _root_=21}, SolrDocument{id=23, intGSL=1050849974, longGSL=4557485844322923262, doubleGSL=23130.060006510375, floatGSL=23433.121, dateGSL=Sun May 22 15:01:58 GMT+14:00 89431712, stringGSL=base_string_815184__00020303, boolGSL=false, sortableGSL=base_string_337557__00021495, _version_=1729380957056663554, _root_=23}, SolrDocument{id=24, intGSL=1050849974, longGSL=4557485844322923262, doubleGSL=23130.060006510375, floatGSL=23433.121, dateGSL=Sun May 22 15:01:58 GMT+14:00 89431712, stringGSL=base_string_815184__00020303, boolGSL=true, sortableGSL=base_string_337557__00021495, _version_=1729380957055614977, _root_=24}, SolrDocument{id=25, intGSL=1050849974, longGSL=4557485844322923262, doubleGSL=23130.060006510375, floatGSL=23433.121, dateGSL=Sun May 22 15:01:58 GMT+14:00 89431712, stringGSL=base_string_815184__00020303, boolGSL=false, sortableGSL=base_string_337557__00021495, _version_=1729380957056663555, _root_=25}, SolrDocument{id=26, intGSL=1050849974, longGSL=4557485844322923262, doubleGSL=23130.060006510375, floatGSL=23433.121, dateGSL=Sun May 22 15:01:58 GMT+14:00 89431712, stringGSL=base_string_815184__00020303, boolGSL=true, sortableGSL=base_string_337557__00021495, _version_=1729380957058760707, _root_=26}, SolrDocument{id=27, intGSL=1050849974, longGSL=4557485844322923262, doubleGSL=23130.060006510375, floatGSL=23433.121, dateGSL=Sun May 22 15:01:58 GMT+14:00 89431712, stringGSL=base_string_815184__00020303, boolGSL=false, sortableGSL=base_string_337557__00021495, _version_=1729380957055614978, _root_=27}]}}, {groupValue=1050858685, doclist={numFound=6,numFoundExact=true,start=0,maxScore=1.0,docs=[SolrDocument{id=28, intGSL=1050858685, longGSL=4557485844322925296, doubleGSL=26608.060006510375, floatGSL=26136.121, dateGSL=Sun May 22 15:02:03 GMT+14:00 89431712, stringGSL=base_string_815184__00022075, boolGSL=true, sortableGSL=base_string_337557__00024212, _version_=1729380957055614979, _root_=28}, SolrDocument{id=29, intGSL=1050858685, longGSL=4557485844322925296, doubleGSL=26608.060006510375, floatGSL=26136.121, dateGSL=Sun May 22 15:02:03 GMT+14:00 89431712, stringGSL=base_string_815184__00022075, boolGSL=false, sortableGSL=base_string_337557__00024212, _version_=1729380957056663556, _root_=29}, SolrDocument{id=30, intGSL=1050858685, longGSL=4557485844322925296, doubleGSL=26608.060006510375, floatGSL=26136.121, dateGSL=Sun May 22 15:02:03 GMT+14:00 89431712, stringGSL=base_string_815184__00022075, boolGSL=true, sortableGSL=base_string_337557__00024212, _version_=1729380957058760708, _root_=30}, SolrDocument{id=31, intGSL=1050858685, longGSL=4557485844322925296, doubleGSL=26608.060006510375, floatGSL=26136.121, dateGSL=Sun May 22 15:02:03 GMT+14:00 89431712, stringGSL=base_string_815184__00022075, boolGSL=false, sortableGSL=base_string_337557__00024212, _version_=1729380957058760709, _root_=31}, SolrDocument{id=32, intGSL=1050858685, longGSL=4557485844322925296, doubleGSL=26608.060006510375, floatGSL=26136.121, dateGSL=Sun May 22 15:02:03 GMT+14:00 89431712, stringGSL=base_string_815184__00022075, boolGSL=true, sortableGSL=base_string_337557__00024212, _version_=1729380957055614980, _root_=32}, SolrDocument{id=33, intGSL=1050858685, longGSL=4557485844322925296, doubleGSL=26608.060006510375, floatGSL=26136.121, dateGSL=Sun May 22 15:02:03 GMT+14:00 89431712, stringGSL=base_string_815184__00022075, boolGSL=false, sortableGSL=base_string_337557__00024212, _version_=1729380957056663558, _root_=33}]}}, {groupValue=1050860925, doclist={numFound=4,numFoundExact=true,start=0,maxScore=1.0,docs=[SolrDocument{id=37, intGSL=1050860925, longGSL=4557485844322927314, doubleGSL=30051.060006510375, floatGSL=26417.121, dateGSL=Sun May 22 15:02:09 GMT+14:00 89431712, stringGSL=base_string_815184__00030670, boolGSL=false, sortableGSL=base_string_337557__00031431, _version_=1729380957060857856, _root_=37}, SolrDocument{id=38, intGSL=1050860925, longGSL=4557485844322927314, doubleGSL=30051.060006510375, floatGSL=26417.121, dateGSL=Sun May 22 15:02:09 GMT+14:00 89431712, stringGSL=base_string_815184__00030670, boolGSL=true, sortableGSL=base_string_337557__00031431, _version_=1729380957055614981, _root_=38}, SolrDocument{id=39, intGSL=1050860925, longGSL=4557485844322927314, doubleGSL=30051.060006510375, floatGSL=26417.121, dateGSL=Sun May 22 15:02:09 GMT+14:00 89431712, stringGSL=base_string_815184__00030670, boolGSL=false, sortableGSL=base_string_337557__00031431, _version_=1729380957058760711, _root_=39}, SolrDocument{id=40, intGSL=1050860925, longGSL=4557485844322927314, doubleGSL=30051.060006510375, floatGSL=26417.121, dateGSL=Sun May 22 15:02:09 GMT+14:00 89431712, stringGSL=base_string_815184__00030670, boolGSL=true, sortableGSL=base_string_337557__00031431, _version_=1729380957056663552, _root_=40}]}}, {groupValue=1050867084, doclist={numFound=3,numFoundExact=true,start=0,maxScore=1.0,docs=[SolrDocument{id=42, intGSL=1050867084, longGSL=4557485844322937265, doubleGSL=37775.06000651037, floatGSL=29443.121, dateGSL=Sun May 22 15:02:10 GMT+14:00 89431712, stringGSL=base_string_815184__00032857, boolGSL=true, sortableGSL=base_string_337557__00033940, _version_=1729380957056663553, _root_=42}, SolrDocument{id=45, intGSL=1050867084, longGSL=4557485844322937265, doubleGSL=37775.06000651037, floatGSL=29443.121, dateGSL=Sun May 22 15:02:10 GMT+14:00 89431712, stringGSL=base_string_815184__00032857, boolGSL=false, sortableGSL=base_string_337557__00033940, _version_=1729380957061906433, _root_=45}, SolrDocument{id=48, intGSL=1050867084, longGSL=4557485844322937265, doubleGSL=37775.06000651037, floatGSL=29443.121, dateGSL=Sun May 22 15:02:10 GMT+14:00 89431712, stringGSL=base_string_815184__00032857, boolGSL=true, sortableGSL=base_string_337557__00033940, _version_=1729380957056663554, _root_=48}]}}, {groupValue=1050846662, doclist={numFound=6,numFoundExact=true,start=0,maxScore=1.0,docs=[SolrDocument{id=10, intGSL=1050846662, longGSL=4557485844322912948, doubleGSL=17896.060006510375, floatGSL=14448.12, dateGSL=Sun May 22 15:01:50 GMT+14:00 89431712, stringGSL=base_string_815184__00011699, boolGSL=true, sortableGSL=base_string_337557__00010270, _version_=1729380957054566401, _root_=10}, SolrDocument{id=11, intGSL=1050846662, longGSL=4557485844322912948, doubleGSL=17896.060006510375, floatGSL=14448.12, dateGSL=Sun May 22 15:01:50 GMT+14:00 89431712, stringGSL=base_string_815184__00011699, boolGSL=false, sortableGSL=base_string_337557__00010270, _version_=1729380957054566402, _root_=11}, SolrDocument{id=13, intGSL=1050846662, longGSL=4557485844322912948, doubleGSL=17896.060006510375, floatGSL=14448.12, dateGSL=Sun May 22 15:01:50 GMT+14:00 89431712, stringGSL=base_string_815184__00011699, boolGSL=false, sortableGSL=base_string_337557__00010270, _version_=1729380957054566403, _root_=13}, SolrDocument{id=7, intGSL=1050846662, longGSL=4557485844322912948, doubleGSL=17896.060006510375, floatGSL=14448.12, dateGSL=Sun May 22 15:01:50 GMT+14:00 89431712, stringGSL=base_string_815184__00011699, boolGSL=false, sortableGSL=base_string_337557__00010270, _version_=1729380957057712130, _root_=7}, SolrDocument{id=8, intGSL=1050846662, longGSL=4557485844322912948, doubleGSL=17896.060006510375, floatGSL=14448.12, dateGSL=Sun May 22 15:01:50 GMT+14:00 89431712, stringGSL=base_string_815184__00011699, boolGSL=true, sortableGSL=base_string_337557__00010270, _version_=1729380957054566400, _root_=8}, SolrDocument{id=9, intGSL=1050846662, longGSL=4557485844322912948, doubleGSL=17896.060006510375, floatGSL=14448.12, dateGSL=Sun May 22 15:01:50 GMT+14:00 89431712, stringGSL=base_string_815184__00011699, boolGSL=false, sortableGSL=base_string_337557__00010270, _version_=1729380957057712131, _root_=9}]}}]}}}
	at __randomizedtesting.SeedInfo.seed([AD68A25AA19271B6:189BE669C03514E8]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.solr.cloud.DocValuesNotIndexedTest.doGroupingDvOnly(DocValuesNotIndexedTest.java:434)
	at org.apache.solr.cloud.DocValuesNotIndexedTest.testGroupingDVOnlySortLast(DocValuesNotIndexedTest.java:361)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:78)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:567)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base/java.lang.Thread.run(Thread.java:831)


FAILED:  org.apache.solr.cloud.TestDistribDocBasedVersion.test

Error Message:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:58906: Underlying core creation failed while creating collection: control_collection

Stack Trace:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:58906: Underlying core creation failed while creating collection: control_collection
	at __randomizedtesting.SeedInfo.seed([AD68A25AA19271B6:253C9D800F6E1C4E]:0)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:715)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:405)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:371)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1248)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:969)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:896)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:223)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:238)
	at app//org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:357)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1162)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1135)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)


FAILED:  org.apache.solr.cloud.TestStressLiveNodes.testStress

Error Message:
java.lang.AssertionError: iter17: [127.0.0.1:57808_solr, thrasher-T16_1-0, thrasher-T16_1-1, thrasher-T16_1-2, thrasher-T16_1-3, thrasher-T16_1-4] expected:<1> but was:<6>

Stack Trace:
java.lang.AssertionError: iter17: [127.0.0.1:57808_solr, thrasher-T16_1-0, thrasher-T16_1-1, thrasher-T16_1-2, thrasher-T16_1-3, thrasher-T16_1-4] expected:<1> but was:<6>
	at __randomizedtesting.SeedInfo.seed([AD68A25AA19271B6:B873DF9B226589CD]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.junit.Assert.failNotEquals(Assert.java:835)
	at org.junit.Assert.assertEquals(Assert.java:647)
	at org.apache.solr.cloud.TestStressLiveNodes.testStress(TestStressLiveNodes.java:144)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:78)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:567)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base/java.lang.Thread.run(Thread.java:831)


FAILED:  org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=1113, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:361)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:75)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getLines(AbstractAnalysisFactory.java:302)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getWordSet(AbstractAnalysisFactory.java:293)
        at app//org.apache.lucene.analysis.en.AbstractWordsFileFilterFactory.inform(AbstractWordsFileFilterFactory.java:88)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:762)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1470)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1319)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1307)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:654)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:188)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:119)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:279)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:345)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:256)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$629/0x0000000801150d08.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:49)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:252)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1850)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3394)
        at app//org.apache.solr.core.SolrCore$$Lambda$739/0x0000000801213378.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2761)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1213/0x0000000801699b20.run(Unknown Source)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=1113, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:361)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:75)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getLines(AbstractAnalysisFactory.java:302)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getWordSet(AbstractAnalysisFactory.java:293)
        at app//org.apache.lucene.analysis.en.AbstractWordsFileFilterFactory.inform(AbstractWordsFileFilterFactory.java:88)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:762)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1470)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1319)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1307)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:654)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:188)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:119)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:279)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:345)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:256)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$629/0x0000000801150d08.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:49)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:252)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1850)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3394)
        at app//org.apache.solr.core.SolrCore$$Lambda$739/0x0000000801213378.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2761)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1213/0x0000000801699b20.run(Unknown Source)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
	at __randomizedtesting.SeedInfo.seed([AD68A25AA19271B6]:0)

[JENKINS] Solr-main-MacOSX (64bit/jdk-11.0.13) - Build # 866 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/866/
Java: 64bit/jdk-11.0.13 -XX:+UseCompressedOops -XX:+UseParallelGC

7 tests failed.
FAILED:  org.apache.solr.schema.SchemaApiFailureTest.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=10359, name=qtp144690366-10359, state=RUNNABLE, group=TGRP-SchemaApiFailureTest], registration stack trace below.

Stack Trace:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=10359, name=qtp144690366-10359, state=RUNNABLE, group=TGRP-SchemaApiFailureTest], registration stack trace below.
	at java.base@11.0.13/java.lang.Thread.getStackTrace(Thread.java:1602)
	at app//com.carrotsearch.randomizedtesting.RandomizedContext.closeAtEnd(RandomizedContext.java:157)
	at app//org.apache.lucene.util.LuceneTestCase.closeAfterSuite(LuceneTestCase.java:798)
	at app//org.apache.lucene.util.LuceneTestCase.wrapDirectory(LuceneTestCase.java:1504)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1368)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1350)
	at app//org.apache.solr.core.MockDirectoryFactory.create(MockDirectoryFactory.java:51)
	at app//org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:368)
	at app//org.apache.solr.core.SolrCore.initSnapshotMetaDataManager(SolrCore.java:606)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1119)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1048)
	at app//org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1562)
	at app//org.apache.solr.core.CoreContainer.create(CoreContainer.java:1428)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:389)
	at app//org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:422)
	at app//org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:205)
	at app//org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:207)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:941)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:893)
	at app//org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:584)
	at app//org.apache.solr.servlet.SolrDispatchFilter.dispatch(SolrDispatchFilter.java:239)
	at app//org.apache.solr.servlet.SolrDispatchFilter.lambda$doFilter$0(SolrDispatchFilter.java:207)
	at app//org.apache.solr.servlet.ServletUtils.traceHttpRequestExecution2(ServletUtils.java:257)
	at app//org.apache.solr.servlet.ServletUtils.rateLimitRequest(ServletUtils.java:227)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:202)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:184)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:187)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1434)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
	at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)
	at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1349)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
	at app//org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:713)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.server.Server.handle(Server.java:516)
	at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:400)
	at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:645)
	at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:392)
	at app//org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:352)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
	at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
Caused by: java.lang.AssertionError: Directory not closed: MockDirectoryWrapper(ByteBuffersDirectory@4081218e lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7ac36c9d)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.lucene.util.CloseableDirectory.close(CloseableDirectory.java:43)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:789)
	at com.carrotsearch.randomizedtesting.RandomizedContext.closeResources(RandomizedContext.java:225)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.afterAlways(RandomizedRunner.java:805)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	... 1 more


FAILED:  org.apache.solr.schema.SchemaApiFailureTest.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=10359, name=qtp144690366-10359, state=RUNNABLE, group=TGRP-SchemaApiFailureTest], registration stack trace below.

Stack Trace:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=10359, name=qtp144690366-10359, state=RUNNABLE, group=TGRP-SchemaApiFailureTest], registration stack trace below.
	at java.base@11.0.13/java.lang.Thread.getStackTrace(Thread.java:1602)
	at app//com.carrotsearch.randomizedtesting.RandomizedContext.closeAtEnd(RandomizedContext.java:157)
	at app//org.apache.lucene.util.LuceneTestCase.closeAfterSuite(LuceneTestCase.java:798)
	at app//org.apache.lucene.util.LuceneTestCase.wrapDirectory(LuceneTestCase.java:1504)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1368)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1350)
	at app//org.apache.solr.core.MockDirectoryFactory.create(MockDirectoryFactory.java:51)
	at app//org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:368)
	at app//org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:412)
	at app//org.apache.solr.core.SolrCore.initIndex(SolrCore.java:856)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1123)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1048)
	at app//org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1562)
	at app//org.apache.solr.core.CoreContainer.create(CoreContainer.java:1428)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:389)
	at app//org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:422)
	at app//org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:205)
	at app//org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:207)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:941)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:893)
	at app//org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:584)
	at app//org.apache.solr.servlet.SolrDispatchFilter.dispatch(SolrDispatchFilter.java:239)
	at app//org.apache.solr.servlet.SolrDispatchFilter.lambda$doFilter$0(SolrDispatchFilter.java:207)
	at app//org.apache.solr.servlet.ServletUtils.traceHttpRequestExecution2(ServletUtils.java:257)
	at app//org.apache.solr.servlet.ServletUtils.rateLimitRequest(ServletUtils.java:227)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:202)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:184)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:187)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1434)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
	at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)
	at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1349)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
	at app//org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:713)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.server.Server.handle(Server.java:516)
	at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:400)
	at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:645)
	at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:392)
	at app//org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:352)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
	at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
Caused by: java.lang.AssertionError: Directory not closed: MockDirectoryWrapper(ByteBuffersDirectory@3492e001 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@45716c7)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.lucene.util.CloseableDirectory.close(CloseableDirectory.java:43)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:789)
	at com.carrotsearch.randomizedtesting.RandomizedContext.closeResources(RandomizedContext.java:225)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.afterAlways(RandomizedRunner.java:805)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	... 1 more


FAILED:  org.apache.solr.schema.SchemaApiFailureTest.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=10359, name=qtp144690366-10359, state=RUNNABLE, group=TGRP-SchemaApiFailureTest], registration stack trace below.

Stack Trace:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=10359, name=qtp144690366-10359, state=RUNNABLE, group=TGRP-SchemaApiFailureTest], registration stack trace below.
	at java.base@11.0.13/java.lang.Thread.getStackTrace(Thread.java:1602)
	at app//com.carrotsearch.randomizedtesting.RandomizedContext.closeAtEnd(RandomizedContext.java:157)
	at app//org.apache.lucene.util.LuceneTestCase.closeAfterSuite(LuceneTestCase.java:798)
	at app//org.apache.lucene.util.LuceneTestCase.wrapDirectory(LuceneTestCase.java:1504)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1368)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1350)
	at app//org.apache.solr.core.MockDirectoryFactory.create(MockDirectoryFactory.java:51)
	at app//org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:368)
	at app//org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:108)
	at app//org.apache.solr.core.SolrCore.initIndex(SolrCore.java:895)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1123)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1048)
	at app//org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1562)
	at app//org.apache.solr.core.CoreContainer.create(CoreContainer.java:1428)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:92)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:389)
	at app//org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:422)
	at app//org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:205)
	at app//org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:207)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:941)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:893)
	at app//org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:584)
	at app//org.apache.solr.servlet.SolrDispatchFilter.dispatch(SolrDispatchFilter.java:239)
	at app//org.apache.solr.servlet.SolrDispatchFilter.lambda$doFilter$0(SolrDispatchFilter.java:207)
	at app//org.apache.solr.servlet.ServletUtils.traceHttpRequestExecution2(ServletUtils.java:257)
	at app//org.apache.solr.servlet.ServletUtils.rateLimitRequest(ServletUtils.java:227)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:202)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:184)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:187)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1434)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
	at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)
	at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1349)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
	at app//org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:713)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.server.Server.handle(Server.java:516)
	at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:400)
	at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:645)
	at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:392)
	at app//org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:352)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
	at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
Caused by: java.lang.AssertionError: Directory not closed: MockDirectoryWrapper(ByteBuffersDirectory@42daaf1a lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4448afb0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.lucene.util.CloseableDirectory.close(CloseableDirectory.java:43)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:789)
	at com.carrotsearch.randomizedtesting.RandomizedContext.closeResources(RandomizedContext.java:225)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.afterAlways(RandomizedRunner.java:805)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	... 1 more


FAILED:  org.apache.solr.schema.SchemaApiFailureTest.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.schema.SchemaApiFailureTest: 
   1) Thread[id=10397, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-SchemaApiFailureTest]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:328)
        at app//org.apache.solr.rest.ManagedResourceStorage$ZooKeeperStorageIO.configure(ManagedResourceStorage.java:242)
        at app//org.apache.solr.rest.ManagedResourceStorage.newStorageIO(ManagedResourceStorage.java:155)
        at app//org.apache.solr.core.SolrCore.initRestManager(SolrCore.java:3207)
        at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1151)
        at app//org.apache.solr.core.SolrCore.reload(SolrCore.java:780)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1857)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3394)
        at app//org.apache.solr.core.SolrCore$$Lambda$833/0x00000008008a1c40.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2761)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1256/0x0000000800cf0440.run(Unknown Source)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   2) Thread[id=10422, name=searcherExecutor-7888-thread-1-processing-127.0.0.1:61227_solr, state=WAITING, group=TGRP-SchemaApiFailureTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:885)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1039)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1345)
        at java.base@11.0.13/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:232)
        at app//org.apache.solr.core.SolrCore.lambda$new$2(SolrCore.java:1142)
        at app//org.apache.solr.core.SolrCore$$Lambda$463/0x00000008005cf840.call(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.FutureTask.run(FutureTask.java:264)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$257/0x000000080049f440.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.schema.SchemaApiFailureTest: 
   1) Thread[id=10397, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-SchemaApiFailureTest]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:328)
        at app//org.apache.solr.rest.ManagedResourceStorage$ZooKeeperStorageIO.configure(ManagedResourceStorage.java:242)
        at app//org.apache.solr.rest.ManagedResourceStorage.newStorageIO(ManagedResourceStorage.java:155)
        at app//org.apache.solr.core.SolrCore.initRestManager(SolrCore.java:3207)
        at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1151)
        at app//org.apache.solr.core.SolrCore.reload(SolrCore.java:780)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1857)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3394)
        at app//org.apache.solr.core.SolrCore$$Lambda$833/0x00000008008a1c40.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2761)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1256/0x0000000800cf0440.run(Unknown Source)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   2) Thread[id=10422, name=searcherExecutor-7888-thread-1-processing-127.0.0.1:61227_solr, state=WAITING, group=TGRP-SchemaApiFailureTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:885)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1039)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1345)
        at java.base@11.0.13/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:232)
        at app//org.apache.solr.core.SolrCore.lambda$new$2(SolrCore.java:1142)
        at app//org.apache.solr.core.SolrCore$$Lambda$463/0x00000008005cf840.call(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.FutureTask.run(FutureTask.java:264)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$257/0x000000080049f440.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
	at __randomizedtesting.SeedInfo.seed([2F680F67C858058E]:0)


FAILED:  org.apache.solr.hdfs.search.TestRecoveryHdfs.testExistOldBufferLog

Error Message:
java.util.concurrent.TimeoutException: Timeout waiting for finish replay updates

Stack Trace:
java.util.concurrent.TimeoutException: Timeout waiting for finish replay updates
	at __randomizedtesting.SeedInfo.seed([2F680F67C858058E:7138123246979507]:0)
	at org.apache.solr.util.TimeOut.waitFor(TimeOut.java:67)
	at org.apache.solr.hdfs.search.TestRecoveryHdfs.testExistOldBufferLog(TestRecoveryHdfs.java:631)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base/java.lang.Thread.run(Thread.java:829)


FAILED:  org.apache.solr.hdfs.search.TestRecoveryHdfs.testCorruptLog

Error Message:
java.lang.RuntimeException: mismatch: '0'!='1' @ response/numFound

Stack Trace:
java.lang.RuntimeException: mismatch: '0'!='1' @ response/numFound
	at __randomizedtesting.SeedInfo.seed([2F680F67C858058E:DDB5BBB432605452]:0)
	at org.apache.solr.SolrTestCaseJ4.assertJQ(SolrTestCaseJ4.java:1137)
	at org.apache.solr.SolrTestCaseJ4.assertJQ(SolrTestCaseJ4.java:1084)
	at org.apache.solr.hdfs.search.TestRecoveryHdfs.testCorruptLog(TestRecoveryHdfs.java:966)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base/java.lang.Thread.run(Thread.java:829)


FAILED:  org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=1191, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:361)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:75)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getLines(AbstractAnalysisFactory.java:302)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getWordSet(AbstractAnalysisFactory.java:293)
        at app//org.apache.lucene.analysis.en.AbstractWordsFileFilterFactory.inform(AbstractWordsFileFilterFactory.java:88)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:762)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1470)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1319)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1307)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:654)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:188)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:119)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:279)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:345)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:256)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$617/0x00000008006b6840.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:49)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:252)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1850)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3394)
        at app//org.apache.solr.core.SolrCore$$Lambda$727/0x000000080077ec40.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2761)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1189/0x0000000800ca5440.run(Unknown Source)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=1191, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:361)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:75)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getLines(AbstractAnalysisFactory.java:302)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getWordSet(AbstractAnalysisFactory.java:293)
        at app//org.apache.lucene.analysis.en.AbstractWordsFileFilterFactory.inform(AbstractWordsFileFilterFactory.java:88)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:762)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1470)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1319)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1307)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:654)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:188)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:119)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:279)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:345)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:256)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$617/0x00000008006b6840.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:49)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:252)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1850)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3394)
        at app//org.apache.solr.core.SolrCore$$Lambda$727/0x000000080077ec40.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2761)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1189/0x0000000800ca5440.run(Unknown Source)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
	at __randomizedtesting.SeedInfo.seed([2F680F67C858058E]:0)

[JENKINS] Solr-main-MacOSX (64bit/jdk-11.0.13) - Build # 865 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/865/
Java: 64bit/jdk-11.0.13 -XX:+UseCompressedOops -XX:+UseParallelGC

5 tests failed.
FAILED:  org.apache.solr.cloud.HttpPartitionOnCommitTest.test

Error Message:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:63921: Underlying core creation failed while creating collection: c8n_2x2_commits

Stack Trace:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:63921: Underlying core creation failed while creating collection: c8n_2x2_commits
	at __randomizedtesting.SeedInfo.seed([20D3807C08C5CFA3:A887BFA6A639A25B]:0)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:715)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at app//org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1193)
	at app//org.apache.solr.cloud.AbstractBasicDistributedZkTestBase.createCollection(AbstractBasicDistributedZkTestBase.java:1177)
	at app//org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:2025)
	at app//org.apache.solr.cloud.HttpPartitionOnCommitTest.multiShardTest(HttpPartitionOnCommitTest.java:78)
	at app//org.apache.solr.cloud.HttpPartitionOnCommitTest.test(HttpPartitionOnCommitTest.java:69)
	at java.base@11.0.13/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base@11.0.13/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base@11.0.13/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base@11.0.13/java.lang.reflect.Method.invoke(Method.java:566)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1164)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1135)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)


FAILED:  org.apache.solr.cloud.ReplaceNodeTest.test

Error Message:
java.lang.AssertionError: Expecting no cores but found some: {replacenodetest_coll_shard3_replica_n21={name=replacenodetest_coll_shard3_replica_n21, instanceDir=/Users/jenkins/workspace/Solr-main-MacOSX/solr/core/build/tmp/tests-tmp/solr.cloud.ReplaceNodeTest_20D3807C08C5CFA3-001/tempDir-002/node6/replacenodetest_coll_shard3_replica_n21, dataDir=/Users/jenkins/workspace/Solr-main-MacOSX/solr/core/build/tmp/tests-tmp/solr.cloud.ReplaceNodeTest_20D3807C08C5CFA3-001/tempDir-002/node6/replacenodetest_coll_shard3_replica_n21/data/, config=solrconfig.xml, schema=schema.xml, startTime=Wed Apr 06 00:00:43 HAST 2022, uptime=33090, lastPublished=active, configVersion=0, cloud={collection=replacenodetest_coll, shard=shard3, replica=core_node22, replicaType=NRT}, index={numDocs=0, maxDoc=0, deletedDocs=0, version=3, segmentCount=0, current=false, hasDeletions=false, directory=org.apache.lucene.store.MockDirectoryWrapper:MockDirectoryWrapper(ByteBuffersDirectory@4a65b17c lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@47391139), segmentsFile=segments_1, segmentsFileSizeInBytes=-1, userData={}, sizeInBytes=135, size=135 bytes}}, replacenodetest_coll_shard5_replica_n23={name=replacenodetest_coll_shard5_replica_n23, instanceDir=/Users/jenkins/workspace/Solr-main-MacOSX/solr/core/build/tmp/tests-tmp/solr.cloud.ReplaceNodeTest_20D3807C08C5CFA3-001/tempDir-002/node6/replacenodetest_coll_shard5_replica_n23, dataDir=/Users/jenkins/workspace/Solr-main-MacOSX/solr/core/build/tmp/tests-tmp/solr.cloud.ReplaceNodeTest_20D3807C08C5CFA3-001/tempDir-002/node6/replacenodetest_coll_shard5_replica_n23/data/, config=solrconfig.xml, schema=schema.xml, startTime=Wed Apr 06 00:00:46 HAST 2022, uptime=29618, lastPublished=active, configVersion=0, cloud={collection=replacenodetest_coll, shard=shard5, replica=core_node24, replicaType=NRT}, index={numDocs=0, maxDoc=0, deletedDocs=0, version=3, segmentCount=0, current=true, hasDeletions=false, directory=org.apache.lucene.store.MockDirectoryWrapper:MockDirectoryWrapper(ByteBuffersDirectory@1bfb1600 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@348be5f7), segmentsFile=segments_1, segmentsFileSizeInBytes=69, userData={}, sizeInBytes=69, size=69 bytes}}} expected:<0> but was:<2>

Stack Trace:
java.lang.AssertionError: Expecting no cores but found some: {replacenodetest_coll_shard3_replica_n21={name=replacenodetest_coll_shard3_replica_n21, instanceDir=/Users/jenkins/workspace/Solr-main-MacOSX/solr/core/build/tmp/tests-tmp/solr.cloud.ReplaceNodeTest_20D3807C08C5CFA3-001/tempDir-002/node6/replacenodetest_coll_shard3_replica_n21, dataDir=/Users/jenkins/workspace/Solr-main-MacOSX/solr/core/build/tmp/tests-tmp/solr.cloud.ReplaceNodeTest_20D3807C08C5CFA3-001/tempDir-002/node6/replacenodetest_coll_shard3_replica_n21/data/, config=solrconfig.xml, schema=schema.xml, startTime=Wed Apr 06 00:00:43 HAST 2022, uptime=33090, lastPublished=active, configVersion=0, cloud={collection=replacenodetest_coll, shard=shard3, replica=core_node22, replicaType=NRT}, index={numDocs=0, maxDoc=0, deletedDocs=0, version=3, segmentCount=0, current=false, hasDeletions=false, directory=org.apache.lucene.store.MockDirectoryWrapper:MockDirectoryWrapper(ByteBuffersDirectory@4a65b17c lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@47391139), segmentsFile=segments_1, segmentsFileSizeInBytes=-1, userData={}, sizeInBytes=135, size=135 bytes}}, replacenodetest_coll_shard5_replica_n23={name=replacenodetest_coll_shard5_replica_n23, instanceDir=/Users/jenkins/workspace/Solr-main-MacOSX/solr/core/build/tmp/tests-tmp/solr.cloud.ReplaceNodeTest_20D3807C08C5CFA3-001/tempDir-002/node6/replacenodetest_coll_shard5_replica_n23, dataDir=/Users/jenkins/workspace/Solr-main-MacOSX/solr/core/build/tmp/tests-tmp/solr.cloud.ReplaceNodeTest_20D3807C08C5CFA3-001/tempDir-002/node6/replacenodetest_coll_shard5_replica_n23/data/, config=solrconfig.xml, schema=schema.xml, startTime=Wed Apr 06 00:00:46 HAST 2022, uptime=29618, lastPublished=active, configVersion=0, cloud={collection=replacenodetest_coll, shard=shard5, replica=core_node24, replicaType=NRT}, index={numDocs=0, maxDoc=0, deletedDocs=0, version=3, segmentCount=0, current=true, hasDeletions=false, directory=org.apache.lucene.store.MockDirectoryWrapper:MockDirectoryWrapper(ByteBuffersDirectory@1bfb1600 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@348be5f7), segmentsFile=segments_1, segmentsFileSizeInBytes=69, userData={}, sizeInBytes=69, size=69 bytes}}} expected:<0> but was:<2>
	at __randomizedtesting.SeedInfo.seed([20D3807C08C5CFA3:A887BFA6A639A25B]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.junit.Assert.failNotEquals(Assert.java:835)
	at org.junit.Assert.assertEquals(Assert.java:647)
	at org.apache.solr.cloud.ReplaceNodeTest.test(ReplaceNodeTest.java:148)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base/java.lang.Thread.run(Thread.java:829)


FAILED:  org.apache.solr.cloud.TestRebalanceLeaders.testRebalanceLeaders

Error Message:
org.apache.solr.client.solrj.SolrServerException: Timeout occurred while waiting response from server at: http://127.0.0.1:56095/solr

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occurred while waiting response from server at: http://127.0.0.1:56095/solr
	at __randomizedtesting.SeedInfo.seed([20D3807C08C5CFA3:3E3CB28C306E83D3]:0)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:724)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:405)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:371)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1248)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:969)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:896)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:223)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:238)
	at app//org.apache.solr.cloud.TestRebalanceLeaders.rebalanceLeaderUsingStandardRequest(TestRebalanceLeaders.java:444)
	at app//org.apache.solr.cloud.TestRebalanceLeaders.sendRebalanceCommand(TestRebalanceLeaders.java:330)
	at app//org.apache.solr.cloud.TestRebalanceLeaders.testRebalanceLeaders(TestRebalanceLeaders.java:145)
	at java.base@11.0.13/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base@11.0.13/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base@11.0.13/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base@11.0.13/java.lang.reflect.Method.invoke(Method.java:566)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.base/java.net.SocketInputStream.socketRead0(Native Method)
	at java.base/java.net.SocketInputStream.socketRead(SocketInputStream.java:115)
	at java.base/java.net.SocketInputStream.read(SocketInputStream.java:168)
	at java.base/java.net.SocketInputStream.read(SocketInputStream.java:140)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:280)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:157)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:576)
	... 58 more


FAILED:  org.apache.solr.security.AuthWithShardHandlerFactoryOverrideTest.collectionTest

Error Message:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:55137/solr: no servers hosting shard: shard4

Stack Trace:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:55137/solr: no servers hosting shard: shard4
	at __randomizedtesting.SeedInfo.seed([20D3807C08C5CFA3:F582865E9B80845E]:0)
	at app//org.apache.solr.client.solrj.impl.Http2SolrClient.processErrorsAndResponse(Http2SolrClient.java:822)
	at app//org.apache.solr.client.solrj.impl.Http2SolrClient.processErrorsAndResponse(Http2SolrClient.java:485)
	at app//org.apache.solr.client.solrj.impl.Http2SolrClient.request(Http2SolrClient.java:453)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:223)
	at app//org.apache.solr.security.AuthWithShardHandlerFactoryOverrideTest.collectionTest(AuthWithShardHandlerFactoryOverrideTest.java:92)
	at java.base@11.0.13/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base@11.0.13/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base@11.0.13/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base@11.0.13/java.lang.reflect.Method.invoke(Method.java:566)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)


FAILED:  org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=1093, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:328)
        at app//org.apache.solr.cloud.ZkController.pathExists(ZkController.java:1228)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:73)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getSnowballWordSet(AbstractAnalysisFactory.java:321)
        at app//org.apache.lucene.analysis.en.AbstractWordsFileFilterFactory.inform(AbstractWordsFileFilterFactory.java:90)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:762)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1470)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1319)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1307)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:654)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:188)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:119)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:279)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:345)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:256)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$633/0x0000000800738c40.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:49)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:252)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1850)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3394)
        at app//org.apache.solr.core.SolrCore$$Lambda$744/0x0000000800806440.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2761)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1189/0x0000000800cbec40.run(Unknown Source)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=1093, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:328)
        at app//org.apache.solr.cloud.ZkController.pathExists(ZkController.java:1228)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:73)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getSnowballWordSet(AbstractAnalysisFactory.java:321)
        at app//org.apache.lucene.analysis.en.AbstractWordsFileFilterFactory.inform(AbstractWordsFileFilterFactory.java:90)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:762)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1470)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1319)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1307)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:654)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:188)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:119)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:279)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:345)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:256)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$633/0x0000000800738c40.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:49)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:252)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1850)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3394)
        at app//org.apache.solr.core.SolrCore$$Lambda$744/0x0000000800806440.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2761)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1189/0x0000000800cbec40.run(Unknown Source)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
	at __randomizedtesting.SeedInfo.seed([20D3807C08C5CFA3]:0)

[JENKINS] Solr-main-MacOSX (64bit/jdk-11.0.13) - Build # 864 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/864/
Java: 64bit/jdk-11.0.13 -XX:+UseCompressedOops -XX:+UseConcMarkSweepGC

4 tests failed.
FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: Timeout occurred while waiting response from server at: http://127.0.0.1:52863/vp/e

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occurred while waiting response from server at: http://127.0.0.1:52863/vp/e
	at __randomizedtesting.SeedInfo.seed([7279FD2F985A5124:FA2DC2F536A63CDC]:0)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:724)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:405)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:371)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1248)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:969)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:896)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:223)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:238)
	at app//org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:433)
	at app//org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:378)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1162)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1135)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.base/java.net.SocketInputStream.socketRead0(Native Method)
	at java.base/java.net.SocketInputStream.socketRead(SocketInputStream.java:115)
	at java.base/java.net.SocketInputStream.read(SocketInputStream.java:168)
	at java.base/java.net.SocketInputStream.read(SocketInputStream.java:140)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:280)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:157)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:576)
	... 51 more


FAILED:  org.apache.solr.cloud.TestRandomRequestDistribution.test

Error Message:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:52881/pm/wq: Underlying core creation failed while creating collection: control_collection

Stack Trace:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:52881/pm/wq: Underlying core creation failed while creating collection: control_collection
	at __randomizedtesting.SeedInfo.seed([7279FD2F985A5124:FA2DC2F536A63CDC]:0)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:715)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:405)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:371)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1248)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:969)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:896)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:223)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:238)
	at app//org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:357)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1162)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1135)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)


FAILED:  org.apache.solr.cloud.api.collections.ShardSplitTest.testSplitAfterFailedSplit

Error Message:
java.lang.AssertionError: Shard split did not succeed after a previous failed split attempt left sub-shards in construction state

Stack Trace:
java.lang.AssertionError: Shard split did not succeed after a previous failed split attempt left sub-shards in construction state
	at __randomizedtesting.SeedInfo.seed([7279FD2F985A5124:8B346E80A42F1CAE]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.solr.cloud.api.collections.ShardSplitTest.splitAfterFailedSplit(ShardSplitTest.java:375)
	at org.apache.solr.cloud.api.collections.ShardSplitTest.testSplitAfterFailedSplit(ShardSplitTest.java:335)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1164)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1135)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base/java.lang.Thread.run(Thread.java:829)


FAILED:  org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=1098, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:361)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:75)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getSnowballWordSet(AbstractAnalysisFactory.java:321)
        at app//org.apache.lucene.analysis.en.AbstractWordsFileFilterFactory.inform(AbstractWordsFileFilterFactory.java:90)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:762)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1470)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1319)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1307)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:654)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:188)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:119)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:279)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:345)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:256)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$621/0x00000008006b4440.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:49)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:252)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1850)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3394)
        at app//org.apache.solr.core.SolrCore$$Lambda$731/0x0000000800776040.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2761)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1175/0x0000000800c66c40.run(Unknown Source)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=1098, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:361)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:75)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getSnowballWordSet(AbstractAnalysisFactory.java:321)
        at app//org.apache.lucene.analysis.en.AbstractWordsFileFilterFactory.inform(AbstractWordsFileFilterFactory.java:90)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:762)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1470)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1319)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1307)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:654)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:188)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:119)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:279)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:345)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:256)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$621/0x00000008006b4440.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:49)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:252)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1850)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3394)
        at app//org.apache.solr.core.SolrCore$$Lambda$731/0x0000000800776040.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2761)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1175/0x0000000800c66c40.run(Unknown Source)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
	at __randomizedtesting.SeedInfo.seed([7279FD2F985A5124]:0)

[JENKINS] Solr-main-MacOSX (64bit/jdk-11.0.13) - Build # 863 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/863/
Java: 64bit/jdk-11.0.13 -XX:+UseCompressedOops -XX:+UseParallelGC

5 tests failed.
FAILED:  org.apache.solr.cloud.PeerSyncReplicationTest.test

Error Message:
org.apache.solr.cloud.ZkController$NotInClusterStateException: Number of replicas in the state does not match what we set:2 vs 3

Stack Trace:
org.apache.solr.cloud.ZkController$NotInClusterStateException: Number of replicas in the state does not match what we set:2 vs 3
	at __randomizedtesting.SeedInfo.seed([43CE1C141D46C4EC:CB9A23CEB3BAA914]:0)
	at app//org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForActiveReplicaCount(AbstractFullDistribZkTestBase.java:709)
	at app//org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:639)
	at app//org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:378)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1162)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1135)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)


FAILED:  org.apache.solr.cloud.PeerSyncReplicationTest.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 106 threads leaked from SUITE scope at org.apache.solr.cloud.PeerSyncReplicationTest: 
   1) Thread[id=10157, name=zkCallback-7062-thread-3, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   2) Thread[id=10166, name=watches-7045-thread-1-processing-127.0.0.1:50408_b_ijl%2Fjt collection1_shard1_replica_n5 collection1 shard1 core_node6, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkStateReader$CacheCleaner.run(ZkStateReader.java:2332)
        at java.base@11.0.13/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
        at java.base@11.0.13/java.util.concurrent.FutureTask.run(FutureTask.java:264)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$253/0x0000000800485040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   3) Thread[id=10057, name=qtp757151583-10057, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   4) Thread[id=10164, name=watches-7064-thread-2, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   5) Thread[id=10062, name=Session-HouseKeeper-3f032bc-1, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at java.base@11.0.13/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@11.0.13/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   6) Thread[id=10078, name=CloserThread, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/java.lang.Object.wait(Native Method)
        at java.base@11.0.13/java.lang.Object.wait(Object.java:328)
        at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2409)
   7) Thread[id=10083, name=qtp798590987-10083, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$204/0x00000008003abc40.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   8) Thread[id=10087, name=qtp798590987-10087, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   9) Thread[id=10167, name=zkCallback-7043-thread-2, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  10) Thread[id=10171, name=searcherExecutor-7084-thread-1, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  11) Thread[id=10133, name=coreContainerWorkExecutor-7053-thread-1, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  12) Thread[id=10142, name=watches-7026-thread-1-processing-127.0.0.1:50401_b_ijl%2Fjt collection1_shard1_replica_n1 collection1 shard1 core_node2, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkStateReader$CacheCleaner.run(ZkStateReader.java:2332)
        at java.base@11.0.13/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
        at java.base@11.0.13/java.util.concurrent.FutureTask.run(FutureTask.java:264)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$253/0x0000000800485040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  13) Thread[id=10217, name=h2sc-7020-thread-25, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  14) Thread[id=10082, name=qtp798590987-10082, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$204/0x00000008003abc40.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  15) Thread[id=10088, name=qtp798590987-10088, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  16) Thread[id=10075, name=zkConnectionManagerCallback-7025-thread-1, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  17) Thread[id=10165, name=Connector-Scheduler-1b5b0ed7-1, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at java.base@11.0.13/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@11.0.13/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  18) Thread[id=10230, name=OverseerStateUpdate-72138414122467338-127.0.0.1:50401_b_ijl%2Fjt-n_0000000001, state=TIMED_WAITING, group=Overseer state updater.]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:488)
        at app//org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:320)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  19) Thread[id=10106, name=coreContainerWorkExecutor-7034-thread-1, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  20) Thread[id=10113, name=qtp659187719-10113, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  21) Thread[id=10112, name=qtp659187719-10112-acceptor-0@345ba4d6-ServerConnector@2ba7fc87{ssl, (ssl, alpn, http/1.1, h2)}{127.0.0.1:50420}, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
        at java.base@11.0.13/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:533)
        at java.base@11.0.13/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:285)
        at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388)
        at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:702)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  22) Thread[id=10172, name=updateExecutor-7038-thread-1, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  23) Thread[id=10179, name=h2sc-7020-thread-7, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  24) Thread[id=10193, name=h2sc-7020-thread-18, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  25) Thread[id=10201, name=h2sc-7020-thread-20, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  26) Thread[id=10192, name=h2sc-7020-thread-17, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  27) Thread[id=10115, name=qtp659187719-10115, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  28) Thread[id=10159, name=updateExecutor-7019-thread-1, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  29) Thread[id=10173, name=recoveryExecutor-7040-thread-1, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  30) Thread[id=10187, name=h2sc-7020-thread-12, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  31) Thread[id=10077, name=zkCallback-7024-thread-1, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  32) Thread[id=10090, name=Session-HouseKeeper-5d0d5524-1, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at java.base@11.0.13/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@11.0.13/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  33) Thread[id=10153, name=searcherExecutor-7077-thread-1, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  34) Thread[id=10146, name=zkCallback-7024-thread-3, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  35) Thread[id=10056, name=qtp757151583-10056, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$204/0x00000008003abc40.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  36) Thread[id=10186, name=h2sc-7020-thread-11, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  37) Thread[id=10086, name=qtp798590987-10086, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  38) Thread[id=10132, name=CloserThread, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/java.lang.Object.wait(Native Method)
        at java.base@11.0.13/java.lang.Object.wait(Object.java:328)
        at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2409)
  39) Thread[id=10114, name=qtp659187719-10114, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  40) Thread[id=10069, name=h2sc-7020-thread-1, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$204/0x00000008003abc40.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$253/0x0000000800485040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  41) Thread[id=10160, name=HttpClient@2bcdbf1a-scheduler-1, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
        at java.base@11.0.13/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  42) Thread[id=10085, name=qtp798590987-10085-acceptor-0@62e22fee-ServerConnector@1b5b0ed7{ssl, (ssl, alpn, http/1.1, h2)}{127.0.0.1:50408}, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
        at java.base@11.0.13/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:533)
        at java.base@11.0.13/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:285)
        at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388)
        at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:702)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  43) Thread[id=10200, name=h2sc-7020-thread-19, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  44) Thread[id=10054, name=qtp757151583-10054, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$204/0x00000008003abc40.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  45) Thread[id=10218, name=h2sc-7020-thread-26, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  46) Thread[id=10203, name=h2sc-7020-thread-22, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  47) Thread[id=10143, name=zkCallback-7024-thread-2, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  48) Thread[id=10109, name=qtp659187719-10109, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$204/0x00000008003abc40.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  49) Thread[id=10060, name=qtp757151583-10060, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  50) Thread[id=10199, name=watches-7045-thread-3, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  51) Thread[id=10116, name=qtp659187719-10116, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  52) Thread[id=10181, name=h2sc-7020-thread-9, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  53) Thread[id=10163, name=h2sc-7020-thread-4, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  54) Thread[id=10151, name=zkCallback-7062-thread-2, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  55) Thread[id=10117, name=Session-HouseKeeper-6a3d9a49-1, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at java.base@11.0.13/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@11.0.13/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  56) Thread[id=10121, name=httpShardExecutor-7056-thread-1, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$204/0x00000008003abc40.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$253/0x0000000800485040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  57) Thread[id=10235, name=zkCallback-7024-thread-5, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  58) Thread[id=10055, name=qtp757151583-10055, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$204/0x00000008003abc40.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  59) Thread[id=10066, name=httpShardExecutor-7018-thread-1, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$204/0x00000008003abc40.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$253/0x0000000800485040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  60) Thread[id=10176, name=updateExecutor-7019-thread-2, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  61) Thread[id=10204, name=h2sc-7020-thread-23, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  62) Thread[id=10231, name=OverseerCollectionConfigSetProcessor-72138414122467338-127.0.0.1:50401_b_ijl%2Fjt-n_0000000001, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.base@11.0.13/java.lang.Object.wait(Native Method)
        at java.base@11.0.13/java.lang.Object.wait(Object.java:328)
        at app//org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1584)
        at app//org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1556)
        at app//org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1970)
        at app//org.apache.solr.common.cloud.SolrZkClient.lambda$getData$6(SolrZkClient.java:361)
        at app//org.apache.solr.common.cloud.SolrZkClient$$Lambda$445/0x000000080060e040.execute(Unknown Source)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:72)
        at app//org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:361)
        at app//org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:487)
        at app//org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:233)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  63) Thread[id=10084, name=qtp798590987-10084, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$204/0x00000008003abc40.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  64) Thread[id=10105, name=CloserThread, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/java.lang.Object.wait(Native Method)
        at java.base@11.0.13/java.lang.Object.wait(Object.java:328)
        at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2409)
  65) Thread[id=10175, name=zkCallback-7043-thread-3, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  66) Thread[id=10124, name=h2sc-7058-thread-1, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$204/0x00000008003abc40.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$253/0x0000000800485040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  67) Thread[id=10168, name=watches-7064-thread-3, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  68) Thread[id=10188, name=h2sc-7020-thread-13, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  69) Thread[id=10162, name=h2sc-7020-thread-3, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  70) Thread[id=10111, name=qtp659187719-10111, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$204/0x00000008003abc40.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  71) Thread[id=10180, name=h2sc-7020-thread-8, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  72) Thread[id=10131, name=zkCallback-7062-thread-1, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  73) Thread[id=10237, name=zkCallback-7062-thread-5, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  74) Thread[id=10154, name=updateExecutor-7057-thread-1, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  75) Thread[id=10061, name=qtp757151583-10061, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  76) Thread[id=10178, name=h2sc-7020-thread-6, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  77) Thread[id=10189, name=h2sc-7020-thread-14, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  78) Thread[id=10149, name=Connector-Scheduler-2ba7fc87-1, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at java.base@11.0.13/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@11.0.13/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  79) Thread[id=10110, name=qtp659187719-10110, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$204/0x00000008003abc40.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  80) Thread[id=10103, name=zkConnectionManagerCallback-7044-thread-1, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  81) Thread[id=10177, name=h2sc-7020-thread-5, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  82) Thread[id=10130, name=zkConnectionManagerCallback-7063-thread-1, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  83) Thread[id=10058, name=qtp757151583-10058-acceptor-0@4aeb33be-ServerConnector@5d3a3251{ssl, (ssl, alpn, http/1.1, h2)}{127.0.0.1:50401}, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
        at java.base@11.0.13/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:533)
        at java.base@11.0.13/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:285)
        at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388)
        at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:702)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  84) Thread[id=10150, name=watches-7064-thread-1-processing-127.0.0.1:50420_b_ijl%2Fjt collection1_shard1_replica_n3 collection1 shard1 core_node4, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkStateReader$CacheCleaner.run(ZkStateReader.java:2332)
        at java.base@11.0.13/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
        at java.base@11.0.13/java.util.concurrent.FutureTask.run(FutureTask.java:264)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$253/0x0000000800485040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  85) Thread[id=10236, name=zkCallback-7062-thread-4, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  86) Thread[id=10141, name=Connector-Scheduler-5d3a3251-1, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at java.base@11.0.13/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@11.0.13/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  87) Thread[id=10145, name=searcherExecutor-7071-thread-1, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  88) Thread[id=10232, name=zkCallback-7043-thread-4, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  89) Thread[id=10148, name=watches-7026-thread-2, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  90) Thread[id=10198, name=watches-7064-thread-4, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  91) Thread[id=10185, name=h2sc-7020-thread-10, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  92) Thread[id=10190, name=h2sc-7020-thread-15, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  93) Thread[id=10059, name=qtp757151583-10059, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  94) Thread[id=10169, name=watches-7026-thread-3, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  95) Thread[id=10161, name=h2sc-7020-thread-2, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  96) Thread[id=10233, name=zkCallback-7043-thread-5, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  97) Thread[id=10089, name=qtp798590987-10089, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  98) Thread[id=10104, name=zkCallback-7043-thread-1, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  99) Thread[id=10184, name=watches-7045-thread-2, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  100) Thread[id=10191, name=h2sc-7020-thread-16, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  101) Thread[id=10234, name=zkCallback-7024-thread-4, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  102) Thread[id=10155, name=recoveryExecutor-7059-thread-1, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  103) Thread[id=10202, name=h2sc-7020-thread-21, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  104) Thread[id=10094, name=httpShardExecutor-7037-thread-1, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$204/0x00000008003abc40.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$253/0x0000000800485040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  105) Thread[id=10097, name=h2sc-7039-thread-1, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$204/0x00000008003abc40.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$253/0x0000000800485040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  106) Thread[id=10205, name=h2sc-7020-thread-24, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 106 threads leaked from SUITE scope at org.apache.solr.cloud.PeerSyncReplicationTest: 
   1) Thread[id=10157, name=zkCallback-7062-thread-3, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   2) Thread[id=10166, name=watches-7045-thread-1-processing-127.0.0.1:50408_b_ijl%2Fjt collection1_shard1_replica_n5 collection1 shard1 core_node6, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkStateReader$CacheCleaner.run(ZkStateReader.java:2332)
        at java.base@11.0.13/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
        at java.base@11.0.13/java.util.concurrent.FutureTask.run(FutureTask.java:264)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$253/0x0000000800485040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   3) Thread[id=10057, name=qtp757151583-10057, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   4) Thread[id=10164, name=watches-7064-thread-2, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   5) Thread[id=10062, name=Session-HouseKeeper-3f032bc-1, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at java.base@11.0.13/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@11.0.13/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   6) Thread[id=10078, name=CloserThread, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/java.lang.Object.wait(Native Method)
        at java.base@11.0.13/java.lang.Object.wait(Object.java:328)
        at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2409)
   7) Thread[id=10083, name=qtp798590987-10083, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$204/0x00000008003abc40.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   8) Thread[id=10087, name=qtp798590987-10087, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   9) Thread[id=10167, name=zkCallback-7043-thread-2, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  10) Thread[id=10171, name=searcherExecutor-7084-thread-1, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  11) Thread[id=10133, name=coreContainerWorkExecutor-7053-thread-1, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  12) Thread[id=10142, name=watches-7026-thread-1-processing-127.0.0.1:50401_b_ijl%2Fjt collection1_shard1_replica_n1 collection1 shard1 core_node2, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkStateReader$CacheCleaner.run(ZkStateReader.java:2332)
        at java.base@11.0.13/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
        at java.base@11.0.13/java.util.concurrent.FutureTask.run(FutureTask.java:264)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$253/0x0000000800485040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  13) Thread[id=10217, name=h2sc-7020-thread-25, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  14) Thread[id=10082, name=qtp798590987-10082, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$204/0x00000008003abc40.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  15) Thread[id=10088, name=qtp798590987-10088, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  16) Thread[id=10075, name=zkConnectionManagerCallback-7025-thread-1, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  17) Thread[id=10165, name=Connector-Scheduler-1b5b0ed7-1, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at java.base@11.0.13/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@11.0.13/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  18) Thread[id=10230, name=OverseerStateUpdate-72138414122467338-127.0.0.1:50401_b_ijl%2Fjt-n_0000000001, state=TIMED_WAITING, group=Overseer state updater.]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:488)
        at app//org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:320)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  19) Thread[id=10106, name=coreContainerWorkExecutor-7034-thread-1, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  20) Thread[id=10113, name=qtp659187719-10113, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  21) Thread[id=10112, name=qtp659187719-10112-acceptor-0@345ba4d6-ServerConnector@2ba7fc87{ssl, (ssl, alpn, http/1.1, h2)}{127.0.0.1:50420}, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
        at java.base@11.0.13/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:533)
        at java.base@11.0.13/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:285)
        at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388)
        at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:702)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  22) Thread[id=10172, name=updateExecutor-7038-thread-1, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  23) Thread[id=10179, name=h2sc-7020-thread-7, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  24) Thread[id=10193, name=h2sc-7020-thread-18, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  25) Thread[id=10201, name=h2sc-7020-thread-20, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  26) Thread[id=10192, name=h2sc-7020-thread-17, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  27) Thread[id=10115, name=qtp659187719-10115, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  28) Thread[id=10159, name=updateExecutor-7019-thread-1, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  29) Thread[id=10173, name=recoveryExecutor-7040-thread-1, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  30) Thread[id=10187, name=h2sc-7020-thread-12, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  31) Thread[id=10077, name=zkCallback-7024-thread-1, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  32) Thread[id=10090, name=Session-HouseKeeper-5d0d5524-1, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at java.base@11.0.13/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@11.0.13/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  33) Thread[id=10153, name=searcherExecutor-7077-thread-1, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  34) Thread[id=10146, name=zkCallback-7024-thread-3, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  35) Thread[id=10056, name=qtp757151583-10056, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$204/0x00000008003abc40.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  36) Thread[id=10186, name=h2sc-7020-thread-11, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  37) Thread[id=10086, name=qtp798590987-10086, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  38) Thread[id=10132, name=CloserThread, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/java.lang.Object.wait(Native Method)
        at java.base@11.0.13/java.lang.Object.wait(Object.java:328)
        at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2409)
  39) Thread[id=10114, name=qtp659187719-10114, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  40) Thread[id=10069, name=h2sc-7020-thread-1, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$204/0x00000008003abc40.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$253/0x0000000800485040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  41) Thread[id=10160, name=HttpClient@2bcdbf1a-scheduler-1, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
        at java.base@11.0.13/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  42) Thread[id=10085, name=qtp798590987-10085-acceptor-0@62e22fee-ServerConnector@1b5b0ed7{ssl, (ssl, alpn, http/1.1, h2)}{127.0.0.1:50408}, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
        at java.base@11.0.13/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:533)
        at java.base@11.0.13/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:285)
        at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388)
        at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:702)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  43) Thread[id=10200, name=h2sc-7020-thread-19, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  44) Thread[id=10054, name=qtp757151583-10054, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$204/0x00000008003abc40.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  45) Thread[id=10218, name=h2sc-7020-thread-26, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  46) Thread[id=10203, name=h2sc-7020-thread-22, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  47) Thread[id=10143, name=zkCallback-7024-thread-2, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  48) Thread[id=10109, name=qtp659187719-10109, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$204/0x00000008003abc40.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  49) Thread[id=10060, name=qtp757151583-10060, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  50) Thread[id=10199, name=watches-7045-thread-3, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  51) Thread[id=10116, name=qtp659187719-10116, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  52) Thread[id=10181, name=h2sc-7020-thread-9, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  53) Thread[id=10163, name=h2sc-7020-thread-4, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  54) Thread[id=10151, name=zkCallback-7062-thread-2, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  55) Thread[id=10117, name=Session-HouseKeeper-6a3d9a49-1, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at java.base@11.0.13/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@11.0.13/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  56) Thread[id=10121, name=httpShardExecutor-7056-thread-1, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$204/0x00000008003abc40.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$253/0x0000000800485040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  57) Thread[id=10235, name=zkCallback-7024-thread-5, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  58) Thread[id=10055, name=qtp757151583-10055, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$204/0x00000008003abc40.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  59) Thread[id=10066, name=httpShardExecutor-7018-thread-1, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$204/0x00000008003abc40.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$253/0x0000000800485040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  60) Thread[id=10176, name=updateExecutor-7019-thread-2, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  61) Thread[id=10204, name=h2sc-7020-thread-23, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  62) Thread[id=10231, name=OverseerCollectionConfigSetProcessor-72138414122467338-127.0.0.1:50401_b_ijl%2Fjt-n_0000000001, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.base@11.0.13/java.lang.Object.wait(Native Method)
        at java.base@11.0.13/java.lang.Object.wait(Object.java:328)
        at app//org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1584)
        at app//org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1556)
        at app//org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1970)
        at app//org.apache.solr.common.cloud.SolrZkClient.lambda$getData$6(SolrZkClient.java:361)
        at app//org.apache.solr.common.cloud.SolrZkClient$$Lambda$445/0x000000080060e040.execute(Unknown Source)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:72)
        at app//org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:361)
        at app//org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:487)
        at app//org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:233)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  63) Thread[id=10084, name=qtp798590987-10084, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$204/0x00000008003abc40.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  64) Thread[id=10105, name=CloserThread, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/java.lang.Object.wait(Native Method)
        at java.base@11.0.13/java.lang.Object.wait(Object.java:328)
        at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2409)
  65) Thread[id=10175, name=zkCallback-7043-thread-3, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  66) Thread[id=10124, name=h2sc-7058-thread-1, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$204/0x00000008003abc40.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$253/0x0000000800485040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  67) Thread[id=10168, name=watches-7064-thread-3, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  68) Thread[id=10188, name=h2sc-7020-thread-13, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  69) Thread[id=10162, name=h2sc-7020-thread-3, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  70) Thread[id=10111, name=qtp659187719-10111, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$204/0x00000008003abc40.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  71) Thread[id=10180, name=h2sc-7020-thread-8, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  72) Thread[id=10131, name=zkCallback-7062-thread-1, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  73) Thread[id=10237, name=zkCallback-7062-thread-5, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  74) Thread[id=10154, name=updateExecutor-7057-thread-1, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  75) Thread[id=10061, name=qtp757151583-10061, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  76) Thread[id=10178, name=h2sc-7020-thread-6, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  77) Thread[id=10189, name=h2sc-7020-thread-14, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  78) Thread[id=10149, name=Connector-Scheduler-2ba7fc87-1, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at java.base@11.0.13/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@11.0.13/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  79) Thread[id=10110, name=qtp659187719-10110, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$204/0x00000008003abc40.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  80) Thread[id=10103, name=zkConnectionManagerCallback-7044-thread-1, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  81) Thread[id=10177, name=h2sc-7020-thread-5, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  82) Thread[id=10130, name=zkConnectionManagerCallback-7063-thread-1, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  83) Thread[id=10058, name=qtp757151583-10058-acceptor-0@4aeb33be-ServerConnector@5d3a3251{ssl, (ssl, alpn, http/1.1, h2)}{127.0.0.1:50401}, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
        at java.base@11.0.13/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:533)
        at java.base@11.0.13/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:285)
        at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388)
        at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:702)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  84) Thread[id=10150, name=watches-7064-thread-1-processing-127.0.0.1:50420_b_ijl%2Fjt collection1_shard1_replica_n3 collection1 shard1 core_node4, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkStateReader$CacheCleaner.run(ZkStateReader.java:2332)
        at java.base@11.0.13/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
        at java.base@11.0.13/java.util.concurrent.FutureTask.run(FutureTask.java:264)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$253/0x0000000800485040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  85) Thread[id=10236, name=zkCallback-7062-thread-4, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  86) Thread[id=10141, name=Connector-Scheduler-5d3a3251-1, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at java.base@11.0.13/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@11.0.13/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  87) Thread[id=10145, name=searcherExecutor-7071-thread-1, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  88) Thread[id=10232, name=zkCallback-7043-thread-4, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  89) Thread[id=10148, name=watches-7026-thread-2, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  90) Thread[id=10198, name=watches-7064-thread-4, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  91) Thread[id=10185, name=h2sc-7020-thread-10, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  92) Thread[id=10190, name=h2sc-7020-thread-15, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  93) Thread[id=10059, name=qtp757151583-10059, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  94) Thread[id=10169, name=watches-7026-thread-3, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  95) Thread[id=10161, name=h2sc-7020-thread-2, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  96) Thread[id=10233, name=zkCallback-7043-thread-5, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  97) Thread[id=10089, name=qtp798590987-10089, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  98) Thread[id=10104, name=zkCallback-7043-thread-1, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  99) Thread[id=10184, name=watches-7045-thread-2, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  100) Thread[id=10191, name=h2sc-7020-thread-16, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  101) Thread[id=10234, name=zkCallback-7024-thread-4, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  102) Thread[id=10155, name=recoveryExecutor-7059-thread-1, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  103) Thread[id=10202, name=h2sc-7020-thread-21, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  104) Thread[id=10094, name=httpShardExecutor-7037-thread-1, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$204/0x00000008003abc40.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$253/0x0000000800485040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  105) Thread[id=10097, name=h2sc-7039-thread-1, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$204/0x00000008003abc40.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$253/0x0000000800485040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  106) Thread[id=10205, name=h2sc-7020-thread-24, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
	at __randomizedtesting.SeedInfo.seed([43CE1C141D46C4EC]:0)


FAILED:  org.apache.solr.cloud.PeerSyncReplicationTest.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=10157, name=zkCallback-7062-thread-3, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   2) Thread[id=10166, name=watches-7045-thread-1, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   3) Thread[id=10057, name=qtp757151583-10057, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   4) Thread[id=10164, name=watches-7064-thread-2, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   5) Thread[id=10062, name=Session-HouseKeeper-3f032bc-1, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at java.base@11.0.13/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@11.0.13/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   6) Thread[id=10078, name=CloserThread, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/java.lang.Object.wait(Native Method)
        at java.base@11.0.13/java.lang.Object.wait(Object.java:328)
        at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2409)
   7) Thread[id=10083, name=qtp798590987-10083, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$204/0x00000008003abc40.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   8) Thread[id=10087, name=qtp798590987-10087, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   9) Thread[id=10167, name=zkCallback-7043-thread-2, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  10) Thread[id=10171, name=searcherExecutor-7084-thread-1, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  11) Thread[id=10133, name=coreContainerWorkExecutor-7053-thread-1, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  12) Thread[id=10142, name=watches-7026-thread-1, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  13) Thread[id=10217, name=h2sc-7020-thread-25, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  14) Thread[id=10082, name=qtp798590987-10082, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$204/0x00000008003abc40.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  15) Thread[id=10088, name=qtp798590987-10088, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  16) Thread[id=10075, name=zkConnectionManagerCallback-7025-thread-1, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  17) Thread[id=10165, name=Connector-Scheduler-1b5b0ed7-1, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
        at java.base@11.0.13/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  18) Thread[id=10106, name=coreContainerWorkExecutor-7034-thread-1, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  19) Thread[id=10113, name=qtp659187719-10113, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  20) Thread[id=10112, name=qtp659187719-10112, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  21) Thread[id=10172, name=updateExecutor-7038-thread-1, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  22) Thread[id=10179, name=h2sc-7020-thread-7, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  23) Thread[id=10193, name=h2sc-7020-thread-18, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  24) Thread[id=10201, name=h2sc-7020-thread-20, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  25) Thread[id=10192, name=h2sc-7020-thread-17, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  26) Thread[id=10115, name=qtp659187719-10115, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  27) Thread[id=10159, name=updateExecutor-7019-thread-1, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  28) Thread[id=10173, name=recoveryExecutor-7040-thread-1, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  29) Thread[id=10187, name=h2sc-7020-thread-12, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  30) Thread[id=10077, name=zkCallback-7024-thread-1, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  31) Thread[id=10090, name=Session-HouseKeeper-5d0d5524-1, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at java.base@11.0.13/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@11.0.13/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  32) Thread[id=10153, name=searcherExecutor-7077-thread-1, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  33) Thread[id=10146, name=zkCallback-7024-thread-3, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  34) Thread[id=10056, name=qtp757151583-10056, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$204/0x00000008003abc40.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  35) Thread[id=10186, name=h2sc-7020-thread-11, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  36) Thread[id=10086, name=qtp798590987-10086, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  37) Thread[id=10132, name=CloserThread, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/java.lang.Object.wait(Native Method)
        at java.base@11.0.13/java.lang.Object.wait(Object.java:328)
        at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2409)
  38) Thread[id=10114, name=qtp659187719-10114, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  39) Thread[id=10069, name=h2sc-7020-thread-1, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$204/0x00000008003abc40.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$253/0x0000000800485040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  40) Thread[id=10160, name=HttpClient@2bcdbf1a-scheduler-1, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
        at java.base@11.0.13/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  41) Thread[id=10085, name=qtp798590987-10085, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  42) Thread[id=10200, name=h2sc-7020-thread-19, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  43) Thread[id=10054, name=qtp757151583-10054, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$204/0x00000008003abc40.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  44) Thread[id=10218, name=h2sc-7020-thread-26, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  45) Thread[id=10203, name=h2sc-7020-thread-22, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  46) Thread[id=10143, name=zkCallback-7024-thread-2, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  47) Thread[id=10109, name=qtp659187719-10109, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$204/0x00000008003abc40.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  48) Thread[id=10060, name=qtp757151583-10060, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  49) Thread[id=10199, name=watches-7045-thread-3, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  50) Thread[id=10116, name=qtp659187719-10116, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  51) Thread[id=10181, name=h2sc-7020-thread-9, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  52) Thread[id=10163, name=h2sc-7020-thread-4, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  53) Thread[id=10151, name=zkCallback-7062-thread-2, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  54) Thread[id=10117, name=Session-HouseKeeper-6a3d9a49-1, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at java.base@11.0.13/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@11.0.13/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  55) Thread[id=10121, name=httpShardExecutor-7056-thread-1, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$204/0x00000008003abc40.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$253/0x0000000800485040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  56) Thread[id=10235, name=zkCallback-7024-thread-5, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  57) Thread[id=10055, name=qtp757151583-10055, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$204/0x00000008003abc40.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  58) Thread[id=10066, name=httpShardExecutor-7018-thread-1, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$204/0x00000008003abc40.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$253/0x0000000800485040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  59) Thread[id=10176, name=updateExecutor-7019-thread-2, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  60) Thread[id=10204, name=h2sc-7020-thread-23, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  61) Thread[id=10084, name=qtp798590987-10084, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$204/0x00000008003abc40.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  62) Thread[id=10105, name=CloserThread, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/java.lang.Object.wait(Native Method)
        at java.base@11.0.13/java.lang.Object.wait(Object.java:328)
        at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2409)
  63) Thread[id=10175, name=zkCallback-7043-thread-3, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  64) Thread[id=10124, name=h2sc-7058-thread-1, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$204/0x00000008003abc40.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$253/0x0000000800485040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  65) Thread[id=10168, name=watches-7064-thread-3, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  66) Thread[id=10188, name=h2sc-7020-thread-13, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  67) Thread[id=10162, name=h2sc-7020-thread-3, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  68) Thread[id=10111, name=qtp659187719-10111, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$204/0x00000008003abc40.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  69) Thread[id=10180, name=h2sc-7020-thread-8, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  70) Thread[id=10131, name=zkCallback-7062-thread-1, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  71) Thread[id=10237, name=zkCallback-7062-thread-5, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  72) Thread[id=10154, name=updateExecutor-7057-thread-1, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  73) Thread[id=10061, name=qtp757151583-10061, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  74) Thread[id=10178, name=h2sc-7020-thread-6, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  75) Thread[id=10189, name=h2sc-7020-thread-14, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  76) Thread[id=10149, name=Connector-Scheduler-2ba7fc87-1, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
        at java.base@11.0.13/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  77) Thread[id=10110, name=qtp659187719-10110, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$204/0x00000008003abc40.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  78) Thread[id=10103, name=zkConnectionManagerCallback-7044-thread-1, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  79) Thread[id=10177, name=h2sc-7020-thread-5, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  80) Thread[id=10130, name=zkConnectionManagerCallback-7063-thread-1, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  81) Thread[id=10058, name=qtp757151583-10058, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  82) Thread[id=10150, name=watches-7064-thread-1, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  83) Thread[id=10236, name=zkCallback-7062-thread-4, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  84) Thread[id=10141, name=Connector-Scheduler-5d3a3251-1, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at java.base@11.0.13/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@11.0.13/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  85) Thread[id=10145, name=searcherExecutor-7071-thread-1, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  86) Thread[id=10232, name=zkCallback-7043-thread-4, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  87) Thread[id=10148, name=watches-7026-thread-2, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  88) Thread[id=10198, name=watches-7064-thread-4, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  89) Thread[id=10185, name=h2sc-7020-thread-10, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  90) Thread[id=10190, name=h2sc-7020-thread-15, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  91) Thread[id=10059, name=qtp757151583-10059, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  92) Thread[id=10169, name=watches-7026-thread-3, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  93) Thread[id=10161, name=h2sc-7020-thread-2, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  94) Thread[id=10233, name=zkCallback-7043-thread-5, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  95) Thread[id=10089, name=qtp798590987-10089, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  96) Thread[id=10104, name=zkCallback-7043-thread-1, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  97) Thread[id=10184, name=watches-7045-thread-2, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  98) Thread[id=10191, name=h2sc-7020-thread-16, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  99) Thread[id=10234, name=zkCallback-7024-thread-4, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  100) Thread[id=10155, name=recoveryExecutor-7059-thread-1, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  101) Thread[id=10202, name=h2sc-7020-thread-21, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  102) Thread[id=10094, name=httpShardExecutor-7037-thread-1, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$204/0x00000008003abc40.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$253/0x0000000800485040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  103) Thread[id=10097, name=h2sc-7039-thread-1, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$204/0x00000008003abc40.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$253/0x0000000800485040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  104) Thread[id=10205, name=h2sc-7020-thread-24, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=10157, name=zkCallback-7062-thread-3, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   2) Thread[id=10166, name=watches-7045-thread-1, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   3) Thread[id=10057, name=qtp757151583-10057, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   4) Thread[id=10164, name=watches-7064-thread-2, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   5) Thread[id=10062, name=Session-HouseKeeper-3f032bc-1, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at java.base@11.0.13/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@11.0.13/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   6) Thread[id=10078, name=CloserThread, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/java.lang.Object.wait(Native Method)
        at java.base@11.0.13/java.lang.Object.wait(Object.java:328)
        at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2409)
   7) Thread[id=10083, name=qtp798590987-10083, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$204/0x00000008003abc40.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   8) Thread[id=10087, name=qtp798590987-10087, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   9) Thread[id=10167, name=zkCallback-7043-thread-2, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  10) Thread[id=10171, name=searcherExecutor-7084-thread-1, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  11) Thread[id=10133, name=coreContainerWorkExecutor-7053-thread-1, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  12) Thread[id=10142, name=watches-7026-thread-1, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  13) Thread[id=10217, name=h2sc-7020-thread-25, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  14) Thread[id=10082, name=qtp798590987-10082, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$204/0x00000008003abc40.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  15) Thread[id=10088, name=qtp798590987-10088, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  16) Thread[id=10075, name=zkConnectionManagerCallback-7025-thread-1, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  17) Thread[id=10165, name=Connector-Scheduler-1b5b0ed7-1, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
        at java.base@11.0.13/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  18) Thread[id=10106, name=coreContainerWorkExecutor-7034-thread-1, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  19) Thread[id=10113, name=qtp659187719-10113, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  20) Thread[id=10112, name=qtp659187719-10112, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  21) Thread[id=10172, name=updateExecutor-7038-thread-1, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  22) Thread[id=10179, name=h2sc-7020-thread-7, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  23) Thread[id=10193, name=h2sc-7020-thread-18, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  24) Thread[id=10201, name=h2sc-7020-thread-20, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  25) Thread[id=10192, name=h2sc-7020-thread-17, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  26) Thread[id=10115, name=qtp659187719-10115, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  27) Thread[id=10159, name=updateExecutor-7019-thread-1, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  28) Thread[id=10173, name=recoveryExecutor-7040-thread-1, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  29) Thread[id=10187, name=h2sc-7020-thread-12, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  30) Thread[id=10077, name=zkCallback-7024-thread-1, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  31) Thread[id=10090, name=Session-HouseKeeper-5d0d5524-1, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at java.base@11.0.13/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@11.0.13/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  32) Thread[id=10153, name=searcherExecutor-7077-thread-1, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  33) Thread[id=10146, name=zkCallback-7024-thread-3, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  34) Thread[id=10056, name=qtp757151583-10056, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$204/0x00000008003abc40.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  35) Thread[id=10186, name=h2sc-7020-thread-11, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  36) Thread[id=10086, name=qtp798590987-10086, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  37) Thread[id=10132, name=CloserThread, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/java.lang.Object.wait(Native Method)
        at java.base@11.0.13/java.lang.Object.wait(Object.java:328)
        at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2409)
  38) Thread[id=10114, name=qtp659187719-10114, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  39) Thread[id=10069, name=h2sc-7020-thread-1, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$204/0x00000008003abc40.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$253/0x0000000800485040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  40) Thread[id=10160, name=HttpClient@2bcdbf1a-scheduler-1, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
        at java.base@11.0.13/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  41) Thread[id=10085, name=qtp798590987-10085, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  42) Thread[id=10200, name=h2sc-7020-thread-19, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  43) Thread[id=10054, name=qtp757151583-10054, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$204/0x00000008003abc40.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  44) Thread[id=10218, name=h2sc-7020-thread-26, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  45) Thread[id=10203, name=h2sc-7020-thread-22, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  46) Thread[id=10143, name=zkCallback-7024-thread-2, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  47) Thread[id=10109, name=qtp659187719-10109, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$204/0x00000008003abc40.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  48) Thread[id=10060, name=qtp757151583-10060, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  49) Thread[id=10199, name=watches-7045-thread-3, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  50) Thread[id=10116, name=qtp659187719-10116, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  51) Thread[id=10181, name=h2sc-7020-thread-9, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  52) Thread[id=10163, name=h2sc-7020-thread-4, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  53) Thread[id=10151, name=zkCallback-7062-thread-2, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  54) Thread[id=10117, name=Session-HouseKeeper-6a3d9a49-1, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at java.base@11.0.13/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@11.0.13/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  55) Thread[id=10121, name=httpShardExecutor-7056-thread-1, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$204/0x00000008003abc40.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$253/0x0000000800485040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  56) Thread[id=10235, name=zkCallback-7024-thread-5, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  57) Thread[id=10055, name=qtp757151583-10055, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$204/0x00000008003abc40.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  58) Thread[id=10066, name=httpShardExecutor-7018-thread-1, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$204/0x00000008003abc40.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$253/0x0000000800485040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  59) Thread[id=10176, name=updateExecutor-7019-thread-2, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  60) Thread[id=10204, name=h2sc-7020-thread-23, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  61) Thread[id=10084, name=qtp798590987-10084, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$204/0x00000008003abc40.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  62) Thread[id=10105, name=CloserThread, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/java.lang.Object.wait(Native Method)
        at java.base@11.0.13/java.lang.Object.wait(Object.java:328)
        at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2409)
  63) Thread[id=10175, name=zkCallback-7043-thread-3, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  64) Thread[id=10124, name=h2sc-7058-thread-1, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$204/0x00000008003abc40.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$253/0x0000000800485040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  65) Thread[id=10168, name=watches-7064-thread-3, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  66) Thread[id=10188, name=h2sc-7020-thread-13, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  67) Thread[id=10162, name=h2sc-7020-thread-3, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  68) Thread[id=10111, name=qtp659187719-10111, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$204/0x00000008003abc40.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  69) Thread[id=10180, name=h2sc-7020-thread-8, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  70) Thread[id=10131, name=zkCallback-7062-thread-1, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  71) Thread[id=10237, name=zkCallback-7062-thread-5, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  72) Thread[id=10154, name=updateExecutor-7057-thread-1, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  73) Thread[id=10061, name=qtp757151583-10061, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  74) Thread[id=10178, name=h2sc-7020-thread-6, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  75) Thread[id=10189, name=h2sc-7020-thread-14, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  76) Thread[id=10149, name=Connector-Scheduler-2ba7fc87-1, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
        at java.base@11.0.13/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  77) Thread[id=10110, name=qtp659187719-10110, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$204/0x00000008003abc40.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  78) Thread[id=10103, name=zkConnectionManagerCallback-7044-thread-1, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  79) Thread[id=10177, name=h2sc-7020-thread-5, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  80) Thread[id=10130, name=zkConnectionManagerCallback-7063-thread-1, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  81) Thread[id=10058, name=qtp757151583-10058, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  82) Thread[id=10150, name=watches-7064-thread-1, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  83) Thread[id=10236, name=zkCallback-7062-thread-4, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  84) Thread[id=10141, name=Connector-Scheduler-5d3a3251-1, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at java.base@11.0.13/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@11.0.13/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  85) Thread[id=10145, name=searcherExecutor-7071-thread-1, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  86) Thread[id=10232, name=zkCallback-7043-thread-4, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  87) Thread[id=10148, name=watches-7026-thread-2, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  88) Thread[id=10198, name=watches-7064-thread-4, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  89) Thread[id=10185, name=h2sc-7020-thread-10, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  90) Thread[id=10190, name=h2sc-7020-thread-15, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  91) Thread[id=10059, name=qtp757151583-10059, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  92) Thread[id=10169, name=watches-7026-thread-3, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  93) Thread[id=10161, name=h2sc-7020-thread-2, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  94) Thread[id=10233, name=zkCallback-7043-thread-5, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  95) Thread[id=10089, name=qtp798590987-10089, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  96) Thread[id=10104, name=zkCallback-7043-thread-1, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  97) Thread[id=10184, name=watches-7045-thread-2, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  98) Thread[id=10191, name=h2sc-7020-thread-16, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  99) Thread[id=10234, name=zkCallback-7024-thread-4, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  100) Thread[id=10155, name=recoveryExecutor-7059-thread-1, state=TIMED_WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  101) Thread[id=10202, name=h2sc-7020-thread-21, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  102) Thread[id=10094, name=httpShardExecutor-7037-thread-1, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$204/0x00000008003abc40.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$253/0x0000000800485040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  103) Thread[id=10097, name=h2sc-7039-thread-1, state=RUNNABLE, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$204/0x00000008003abc40.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$253/0x0000000800485040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  104) Thread[id=10205, name=h2sc-7020-thread-24, state=WAITING, group=TGRP-PeerSyncReplicationTest]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at app//org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:342)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
	at __randomizedtesting.SeedInfo.seed([43CE1C141D46C4EC]:0)


FAILED:  org.apache.solr.hdfs.search.TestRecoveryHdfs.testExistOldBufferLog

Error Message:
java.util.concurrent.TimeoutException: Timeout waiting for finish replay updates

Stack Trace:
java.util.concurrent.TimeoutException: Timeout waiting for finish replay updates
	at __randomizedtesting.SeedInfo.seed([43CE1C141D46C4EC:1D9E014193895465]:0)
	at org.apache.solr.util.TimeOut.waitFor(TimeOut.java:67)
	at org.apache.solr.hdfs.search.TestRecoveryHdfs.testExistOldBufferLog(TestRecoveryHdfs.java:631)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base/java.lang.Thread.run(Thread.java:829)


FAILED:  org.apache.solr.hdfs.search.TestRecoveryHdfs.testRecoveryMultipleLogs

Error Message:
java.lang.RuntimeException: mismatch: '6'!='7' @ response/numFound

Stack Trace:
java.lang.RuntimeException: mismatch: '6'!='7' @ response/numFound
	at __randomizedtesting.SeedInfo.seed([43CE1C141D46C4EC:FA5B84259A3D8305]:0)
	at org.apache.solr.SolrTestCaseJ4.assertJQ(SolrTestCaseJ4.java:1137)
	at org.apache.solr.SolrTestCaseJ4.assertJQ(SolrTestCaseJ4.java:1084)
	at org.apache.solr.hdfs.search.TestRecoveryHdfs.testRecoveryMultipleLogs(TestRecoveryHdfs.java:1077)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base/java.lang.Thread.run(Thread.java:829)

[JENKINS] Solr-main-MacOSX (64bit/jdk-17.0.2) - Build # 862 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/862/
Java: 64bit/jdk-17.0.2 -XX:+UseCompressedOops -XX:+UseShenandoahGC

4 tests failed.
FAILED:  org.apache.solr.handler.designer.TestSchemaDesignerAPI.testAddTechproductsProgressively

Error Message:
org.apache.solr.client.solrj.SolrServerException: Timeout occurred while waiting response from server at: https://127.0.0.1:65490/solr/._designer_techproducts

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occurred while waiting response from server at: https://127.0.0.1:65490/solr/._designer_techproducts
	at __randomizedtesting.SeedInfo.seed([DCA18D3CF0EA6134:A94F9583AE69932A]:0)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:724)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:405)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:371)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1248)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:969)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:896)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:223)
	at app//org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:871)
	at app//org.apache.solr.handler.designer.SchemaDesignerAPI.indexSampleDocs(SchemaDesignerAPI.java:1044)
	at app//org.apache.solr.handler.designer.SchemaDesignerAPI.indexSampleDocsWithRebuildOnAnalysisError(SchemaDesignerAPI.java:1013)
	at app//org.apache.solr.handler.designer.SchemaDesignerAPI.analyze(SchemaDesignerAPI.java:738)
	at app//org.apache.solr.handler.designer.TestSchemaDesignerAPI.testAddTechproductsProgressively(TestSchemaDesignerAPI.java:199)
	at java.base@17.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base@17.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base@17.0.2/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base@17.0.2/java.lang.reflect.Method.invoke(Method.java:568)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.base/sun.nio.ch.NioSocketImpl.timedRead(NioSocketImpl.java:283)
	at java.base/sun.nio.ch.NioSocketImpl.implRead(NioSocketImpl.java:309)
	at java.base/sun.nio.ch.NioSocketImpl.read(NioSocketImpl.java:350)
	at java.base/sun.nio.ch.NioSocketImpl$1.read(NioSocketImpl.java:803)
	at java.base/java.net.Socket$SocketInputStream.read(Socket.java:966)
	at java.base/sun.security.ssl.SSLSocketInputRecord.read(SSLSocketInputRecord.java:478)
	at java.base/sun.security.ssl.SSLSocketInputRecord.readHeader(SSLSocketInputRecord.java:472)
	at java.base/sun.security.ssl.SSLSocketInputRecord.bytesInCompletePacket(SSLSocketInputRecord.java:70)
	at java.base/sun.security.ssl.SSLSocketImpl.readApplicationRecord(SSLSocketImpl.java:1455)
	at java.base/sun.security.ssl.SSLSocketImpl$AppInputStream.read(SSLSocketImpl.java:1059)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:280)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:157)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
	at org.apache.solr.util.stats.InstrumentedHttpRequestExecutor.execute(InstrumentedHttpRequestExecutor.java:142)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:576)
	... 59 more


FAILED:  org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=1140, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@17.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:328)
        at app//org.apache.solr.cloud.ZkController.pathExists(ZkController.java:1228)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:73)
        at app//org.apache.lucene.analysis.synonym.SynonymGraphFilterFactory.loadSynonyms(SynonymGraphFilterFactory.java:191)
        at app//org.apache.lucene.analysis.synonym.SynonymGraphFilterFactory.inform(SynonymGraphFilterFactory.java:161)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:762)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1470)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1328)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1307)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:654)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:188)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:119)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:279)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:345)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:256)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$633/0x0000000801156a18.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:49)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:252)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1850)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3394)
        at app//org.apache.solr.core.SolrCore$$Lambda$743/0x000000080121c000.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2761)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1208/0x000000080167eb40.run(Unknown Source)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=1140, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@17.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:328)
        at app//org.apache.solr.cloud.ZkController.pathExists(ZkController.java:1228)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:73)
        at app//org.apache.lucene.analysis.synonym.SynonymGraphFilterFactory.loadSynonyms(SynonymGraphFilterFactory.java:191)
        at app//org.apache.lucene.analysis.synonym.SynonymGraphFilterFactory.inform(SynonymGraphFilterFactory.java:161)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:762)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1470)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1328)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1307)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:654)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:188)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:119)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:279)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:345)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:256)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$633/0x0000000801156a18.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:49)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:252)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1850)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3394)
        at app//org.apache.solr.core.SolrCore$$Lambda$743/0x000000080121c000.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2761)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1208/0x000000080167eb40.run(Unknown Source)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
	at __randomizedtesting.SeedInfo.seed([DCA18D3CF0EA6134]:0)


FAILED:  org.apache.solr.schema.TestBulkSchemaConcurrent.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.schema.TestBulkSchemaConcurrent: 
   1) Thread[id=10946, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestBulkSchemaConcurrent]
        at java.base@17.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:328)
        at app//org.apache.solr.rest.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream(ManagedResourceStorage.java:275)
        at app//org.apache.solr.rest.ManagedResourceStorage.load(ManagedResourceStorage.java:505)
        at app//org.apache.solr.rest.RestManager$RestManagerManagedResource.reloadFromStorage(RestManager.java:426)
        at app//org.apache.solr.rest.ManagedResource.loadManagedDataAndNotify(ManagedResource.java:81)
        at app//org.apache.solr.rest.RestManager.init(RestManager.java:580)
        at app//org.apache.solr.core.SolrCore.initRestManager(SolrCore.java:3208)
        at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1151)
        at app//org.apache.solr.core.SolrCore.reload(SolrCore.java:780)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1857)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3394)
        at app//org.apache.solr.core.SolrCore$$Lambda$739/0x00000008012a4a38.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2761)
        at app//org.apache.solr.cloud.ZkController$$Lambda$857/0x00000008012f2b80.run(Unknown Source)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   2) Thread[id=11056, name=searcherExecutor-7798-thread-1-processing-127.0.0.1:55629_t_, state=WAITING, group=TGRP-TestBulkSchemaConcurrent]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
        at java.base@17.0.2/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:230)
        at app//org.apache.solr.core.SolrCore.lambda$new$2(SolrCore.java:1142)
        at app//org.apache.solr.core.SolrCore$$Lambda$713/0x000000080128cb10.call(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.FutureTask.run(FutureTask.java:264)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$241/0x0000000800edfcd0.run(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.schema.TestBulkSchemaConcurrent: 
   1) Thread[id=10946, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestBulkSchemaConcurrent]
        at java.base@17.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:161)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:82)
        at app//org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:328)
        at app//org.apache.solr.rest.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream(ManagedResourceStorage.java:275)
        at app//org.apache.solr.rest.ManagedResourceStorage.load(ManagedResourceStorage.java:505)
        at app//org.apache.solr.rest.RestManager$RestManagerManagedResource.reloadFromStorage(RestManager.java:426)
        at app//org.apache.solr.rest.ManagedResource.loadManagedDataAndNotify(ManagedResource.java:81)
        at app//org.apache.solr.rest.RestManager.init(RestManager.java:580)
        at app//org.apache.solr.core.SolrCore.initRestManager(SolrCore.java:3208)
        at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1151)
        at app//org.apache.solr.core.SolrCore.reload(SolrCore.java:780)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1857)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3394)
        at app//org.apache.solr.core.SolrCore$$Lambda$739/0x00000008012a4a38.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2761)
        at app//org.apache.solr.cloud.ZkController$$Lambda$857/0x00000008012f2b80.run(Unknown Source)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   2) Thread[id=11056, name=searcherExecutor-7798-thread-1-processing-127.0.0.1:55629_t_, state=WAITING, group=TGRP-TestBulkSchemaConcurrent]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
        at java.base@17.0.2/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:230)
        at app//org.apache.solr.core.SolrCore.lambda$new$2(SolrCore.java:1142)
        at app//org.apache.solr.core.SolrCore$$Lambda$713/0x000000080128cb10.call(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.FutureTask.run(FutureTask.java:264)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:259)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$241/0x0000000800edfcd0.run(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
	at __randomizedtesting.SeedInfo.seed([DCA18D3CF0EA6134]:0)


FAILED:  org.apache.solr.hdfs.cloud.HdfsThreadLeakTest.testBasic

Error Message:
java.lang.IllegalArgumentException: Self-suppression not permitted

Stack Trace:
java.lang.IllegalArgumentException: Self-suppression not permitted
	at __randomizedtesting.SeedInfo.seed([DCA18D3CF0EA6134:775B90292F36E71A]:0)
	at java.base/java.lang.Throwable.addSuppressed(Throwable.java:1072)
	at org.apache.solr.hdfs.cloud.HdfsThreadLeakTest.testBasic(HdfsThreadLeakTest.java:69)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:568)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base/java.lang.Thread.run(Thread.java:833)
Caused by: org.apache.hadoop.ipc.RemoteException(java.io.IOException): File /testfile could only be written to 0 of the 1 minReplication nodes. There are 1 datanode(s) running and 1 node(s) are excluded in this operation.
	at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.chooseTarget4NewBlock(BlockManager.java:2315)
	at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.chooseTargetForNewBlock(FSDirWriteFileOp.java:294)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAdditionalBlock(FSNamesystem.java:2960)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.addBlock(NameNodeRpcServer.java:904)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.addBlock(ClientNamenodeProtocolServerSideTranslatorPB.java:593)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:604)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:572)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:556)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1093)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1043)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:971)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1878)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2976)

	at app//org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1612)
	at app//org.apache.hadoop.ipc.Client.call(Client.java:1558)
	at app//org.apache.hadoop.ipc.Client.call(Client.java:1455)
	at app//org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:242)
	at app//org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:129)
	at app/jdk.proxy3/jdk.proxy3.$Proxy72.addBlock(Unknown Source)
	at app//org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.addBlock(ClientNamenodeProtocolTranslatorPB.java:530)
	at java.base@17.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base@17.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base@17.0.2/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base@17.0.2/java.lang.reflect.Method.invoke(Method.java:568)
	at app//org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:422)
	at app//org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:165)
	at app//org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:157)
	at app//org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:95)
	at app//org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:359)
	at app/jdk.proxy3/jdk.proxy3.$Proxy73.addBlock(Unknown Source)
	at app//org.apache.hadoop.hdfs.DFSOutputStream.addBlock(DFSOutputStream.java:1088)
	at app//org.apache.hadoop.hdfs.DataStreamer.locateFollowingBlock(DataStreamer.java:1915)
	at app//org.apache.hadoop.hdfs.DataStreamer.nextBlockOutputStream(DataStreamer.java:1717)
	at app//org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:713)

[JENKINS] Solr-main-MacOSX (64bit/jdk-17.0.2) - Build # 861 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/861/
Java: 64bit/jdk-17.0.2 -XX:+UseCompressedOops -XX:+UseShenandoahGC

2 tests failed.
FAILED:  org.apache.solr.cloud.ReplaceNodeTest.testGoodSpreadDuringAssignWithNoTarget

Error Message:
java.lang.AssertionError: There should be no more replicas on the sourceNode after a replaceNode request. expected:<[]> but was:<[core_node15:{
  "core":"replacenodetest_coll_shard1_replica_n12",
  "core_node_name":"core_node15",
  "node_name":"127.0.0.1:53247_solr",
  "base_url":"http://127.0.0.1:53247/solr",
  "state":"active",
  "collection":"replacenodetest_coll",
  "shard":"shard1",
  "type":"NRT",
  "force_set_state":"false"}, core_node18:{
  "core":"replacenodetest_coll_shard2_replica_n16",
  "core_node_name":"core_node18",
  "node_name":"127.0.0.1:53247_solr",
  "base_url":"http://127.0.0.1:53247/solr",
  "state":"active",
  "collection":"replacenodetest_coll",
  "shard":"shard2",
  "type":"NRT",
  "force_set_state":"false"}, core_node13:{
  "core":"replacenodetest_coll_shard3_replica_n10",
  "core_node_name":"core_node13",
  "node_name":"127.0.0.1:53247_solr",
  "base_url":"http://127.0.0.1:53247/solr",
  "state":"active",
  "collection":"replacenodetest_coll",
  "shard":"shard3",
  "type":"NRT",
  "force_set_state":"false"}, core_node23:{
  "core":"replacenodetest_coll_shard4_replica_n20",
  "core_node_name":"core_node23",
  "node_name":"127.0.0.1:53247_solr",
  "base_url":"http://127.0.0.1:53247/solr",
  "state":"active",
  "collection":"replacenodetest_coll",
  "shard":"shard4",
  "type":"NRT",
  "force_set_state":"false"}]>

Stack Trace:
java.lang.AssertionError: There should be no more replicas on the sourceNode after a replaceNode request. expected:<[]> but was:<[core_node15:{
  "core":"replacenodetest_coll_shard1_replica_n12",
  "core_node_name":"core_node15",
  "node_name":"127.0.0.1:53247_solr",
  "base_url":"http://127.0.0.1:53247/solr",
  "state":"active",
  "collection":"replacenodetest_coll",
  "shard":"shard1",
  "type":"NRT",
  "force_set_state":"false"}, core_node18:{
  "core":"replacenodetest_coll_shard2_replica_n16",
  "core_node_name":"core_node18",
  "node_name":"127.0.0.1:53247_solr",
  "base_url":"http://127.0.0.1:53247/solr",
  "state":"active",
  "collection":"replacenodetest_coll",
  "shard":"shard2",
  "type":"NRT",
  "force_set_state":"false"}, core_node13:{
  "core":"replacenodetest_coll_shard3_replica_n10",
  "core_node_name":"core_node13",
  "node_name":"127.0.0.1:53247_solr",
  "base_url":"http://127.0.0.1:53247/solr",
  "state":"active",
  "collection":"replacenodetest_coll",
  "shard":"shard3",
  "type":"NRT",
  "force_set_state":"false"}, core_node23:{
  "core":"replacenodetest_coll_shard4_replica_n20",
  "core_node_name":"core_node23",
  "node_name":"127.0.0.1:53247_solr",
  "base_url":"http://127.0.0.1:53247/solr",
  "state":"active",
  "collection":"replacenodetest_coll",
  "shard":"shard4",
  "type":"NRT",
  "force_set_state":"false"}]>
	at __randomizedtesting.SeedInfo.seed([E20551A048AC9552:CC6E859A7DCF4426]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.junit.Assert.failNotEquals(Assert.java:835)
	at org.junit.Assert.assertEquals(Assert.java:120)
	at org.apache.solr.cloud.ReplaceNodeTest.testGoodSpreadDuringAssignWithNoTarget(ReplaceNodeTest.java:272)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:568)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:824)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:475)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:375)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:831)
	at java.base/java.lang.Thread.run(Thread.java:833)


FAILED:  org.apache.solr.response.TestRawTransformer.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.response.TestRawTransformer: 
   1) Thread[id=3732, name=ConnnectionExpirer, state=TIMED_WAITING, group=TGRP-TestRawTransformer]
        at java.base@17.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.server.NIOServerCnxnFactory$ConnectionExpirerThread.run(NIOServerCnxnFactory.java:554)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.response.TestRawTransformer: 
   1) Thread[id=3732, name=ConnnectionExpirer, state=TIMED_WAITING, group=TGRP-TestRawTransformer]
        at java.base@17.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.server.NIOServerCnxnFactory$ConnectionExpirerThread.run(NIOServerCnxnFactory.java:554)
	at __randomizedtesting.SeedInfo.seed([E20551A048AC9552]:0)