You are viewing a plain text version of this content. The canonical link for it is here.
Posted to builds@solr.apache.org by Policeman Jenkins Server via builds <bu...@solr.apache.org> on 2026/03/08 11:26:08 UTC

[JENKINS] Solr-main-Linux (64bit/hotspot/jdk-23.0.2) - Build # 32036 - Still Unstable!

Build: https://jenkins.thetaphi.de/job/Solr-main-Linux/32036/
Java: 64bit/hotspot/jdk-23.0.2 -XX:+UseCompressedOops -XX:+UseSerialGC

3 tests failed.
FAILED:  org.apache.solr.cloud.BasicDistributedZkTest.classMethod

Error Message:
java.lang.AssertionError: ObjectTracker found 2 object(s) that were not released!!! [HttpJettySolrClient, HttpJettySolrClient]
org.apache.solr.client.solrj.jetty.HttpJettySolrClient:org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.client.solrj.jetty.HttpJettySolrClient
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:54)
	at org.apache.solr.client.solrj.jetty.HttpJettySolrClient.<init>(HttpJettySolrClient.java:177)
	at org.apache.solr.client.solrj.jetty.HttpJettySolrClient$Builder.build(HttpJettySolrClient.java:983)
	at org.apache.solr.update.UpdateShardHandler.<init>(UpdateShardHandler.java:119)
	at org.apache.solr.core.CoreContainer.loadInternal(CoreContainer.java:793)
	at org.apache.solr.core.CoreContainer.load(CoreContainer.java:753)
	at org.apache.solr.servlet.CoreContainerProvider.createCoreContainer(CoreContainerProvider.java:343)
	at org.apache.solr.servlet.CoreContainerProvider.init(CoreContainerProvider.java:182)
	at org.apache.solr.servlet.CoreContainerProvider.contextInitialized(CoreContainerProvider.java:83)
	at org.apache.solr.embedded.JettySolrRunner$1.contextInitialized(JettySolrRunner.java:395)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.callContextInitialized(ServletContextHandler.java:1591)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.contextInitialized(ServletContextHandler.java:500)
	at org.eclipse.jetty.ee10.servlet.ServletHandler.initialize(ServletHandler.java:676)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.startContext(ServletContextHandler.java:1325)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.lambda$doStart$0(ServletContextHandler.java:1051)
	at org.eclipse.jetty.server.handler.ContextHandler$ScopedContext.call(ContextHandler.java:1463)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.doStart(ServletContextHandler.java:1048)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:120)
	at org.eclipse.jetty.server.Handler$Abstract.doStart(Handler.java:491)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:120)
	at org.eclipse.jetty.server.Handler$Abstract.doStart(Handler.java:491)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.doStart(GzipHandler.java:132)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:120)
	at org.eclipse.jetty.server.Handler$Abstract.doStart(Handler.java:491)
	at org.eclipse.jetty.server.handler.GracefulHandler.doStart(GracefulHandler.java:134)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
	at org.eclipse.jetty.server.Server.start(Server.java:643)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:120)
	at org.eclipse.jetty.server.Handler$Abstract.doStart(Handler.java:491)
	at org.eclipse.jetty.server.Server.doStart(Server.java:584)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.apache.solr.embedded.JettySolrRunner.retryOnPortBindFailure(JettySolrRunner.java:628)
	at org.apache.solr.embedded.JettySolrRunner.start(JettySolrRunner.java:574)
	at org.apache.solr.embedded.JettySolrRunner.start(JettySolrRunner.java:546)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.lambda$createJettys$1(AbstractFullDistribZkTestBase.java:559)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:381)
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
	at java.base/java.lang.Thread.run(Thread.java:1575)
Caused by: java.lang.Exception: Submitter stack trace
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.execute(ExecutorUtil.java:348)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:548)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:408)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1060)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1033)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
	... 1 more

org.apache.solr.client.solrj.jetty.HttpJettySolrClient:org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.client.solrj.jetty.HttpJettySolrClient
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:54)
	at org.apache.solr.client.solrj.jetty.HttpJettySolrClient.<init>(HttpJettySolrClient.java:177)
	at org.apache.solr.client.solrj.jetty.HttpJettySolrClient$Builder.build(HttpJettySolrClient.java:983)
	at org.apache.solr.update.UpdateShardHandler.<init>(UpdateShardHandler.java:121)
	at org.apache.solr.core.CoreContainer.loadInternal(CoreContainer.java:793)
	at org.apache.solr.core.CoreContainer.load(CoreContainer.java:753)
	at org.apache.solr.servlet.CoreContainerProvider.createCoreContainer(CoreContainerProvider.java:343)
	at org.apache.solr.servlet.CoreContainerProvider.init(CoreContainerProvider.java:182)
	at org.apache.solr.servlet.CoreContainerProvider.contextInitialized(CoreContainerProvider.java:83)
	at org.apache.solr.embedded.JettySolrRunner$1.contextInitialized(JettySolrRunner.java:395)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.callContextInitialized(ServletContextHandler.java:1591)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.contextInitialized(ServletContextHandler.java:500)
	at org.eclipse.jetty.ee10.servlet.ServletHandler.initialize(ServletHandler.java:676)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.startContext(ServletContextHandler.java:1325)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.lambda$doStart$0(ServletContextHandler.java:1051)
	at org.eclipse.jetty.server.handler.ContextHandler$ScopedContext.call(ContextHandler.java:1463)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.doStart(ServletContextHandler.java:1048)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:120)
	at org.eclipse.jetty.server.Handler$Abstract.doStart(Handler.java:491)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:120)
	at org.eclipse.jetty.server.Handler$Abstract.doStart(Handler.java:491)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.doStart(GzipHandler.java:132)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:120)
	at org.eclipse.jetty.server.Handler$Abstract.doStart(Handler.java:491)
	at org.eclipse.jetty.server.handler.GracefulHandler.doStart(GracefulHandler.java:134)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
	at org.eclipse.jetty.server.Server.start(Server.java:643)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:120)
	at org.eclipse.jetty.server.Handler$Abstract.doStart(Handler.java:491)
	at org.eclipse.jetty.server.Server.doStart(Server.java:584)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.apache.solr.embedded.JettySolrRunner.retryOnPortBindFailure(JettySolrRunner.java:628)
	at org.apache.solr.embedded.JettySolrRunner.start(JettySolrRunner.java:574)
	at org.apache.solr.embedded.JettySolrRunner.start(JettySolrRunner.java:546)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.lambda$createJettys$1(AbstractFullDistribZkTestBase.java:559)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:381)
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
	at java.base/java.lang.Thread.run(Thread.java:1575)
Caused by: java.lang.Exception: Submitter stack trace
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.execute(ExecutorUtil.java:348)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:548)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:408)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1060)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1033)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
	... 1 more

 expected null, but was:<ObjectTracker found 2 object(s) that were not released!!! [HttpJettySolrClient, HttpJettySolrClient]
org.apache.solr.client.solrj.jetty.HttpJettySolrClient:org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.client.solrj.jetty.HttpJettySolrClient
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:54)
	at org.apache.solr.client.solrj.jetty.HttpJettySolrClient.<init>(HttpJettySolrClient.java:177)
	at org.apache.solr.client.solrj.jetty.HttpJettySolrClient$Builder.build(HttpJettySolrClient.java:983)
	at org.apache.solr.update.UpdateShardHandler.<init>(UpdateShardHandler.java:119)
	at org.apache.solr.core.CoreContainer.loadInternal(CoreContainer.java:793)
	at org.apache.solr.core.CoreContainer.load(CoreContainer.java:753)
	at org.apache.solr.servlet.CoreContainerProvider.createCoreContainer(CoreContainerProvider.java:343)
	at org.apache.solr.servlet.CoreContainerProvider.init(CoreContainerProvider.java:182)
	at org.apache.solr.servlet.CoreContainerProvider.contextInitialized(CoreContainerProvider.java:83)
	at org.apache.solr.embedded.JettySolrRunner$1.contextInitialized(JettySolrRunner.java:395)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.callContextInitialized(ServletContextHandler.java:1591)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.contextInitialized(ServletContextHandler.java:500)
	at org.eclipse.jetty.ee10.servlet.ServletHandler.initialize(ServletHandler.java:676)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.startContext(ServletContextHandler.java:1325)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.lambda$doStart$0(ServletContextHandler.java:1051)
	at org.eclipse.jetty.server.handler.ContextHandler$ScopedContext.call(ContextHandler.java:1463)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.doStart(ServletContextHandler.java:1048)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:120)
	at org.eclipse.jetty.server.Handler$Abstract.doStart(Handler.java:491)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:120)
	at org.eclipse.jetty.server.Handler$Abstract.doStart(Handler.java:491)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.doStart(GzipHandler.java:132)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:120)
	at org.eclipse.jetty.server.Handler$Abstract.doStart(Handler.java:491)
	at org.eclipse.jetty.server.handler.GracefulHandler.doStart(GracefulHandler.java:134)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
	at org.eclipse.jetty.server.Server.start(Server.java:643)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:120)
	at org.eclipse.jetty.server.Handler$Abstract.doStart(Handler.java:491)
	at org.eclipse.jetty.server.Server.doStart(Server.java:584)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.apache.solr.embedded.JettySolrRunner.retryOnPortBindFailure(JettySolrRunner.java:628)
	at org.apache.solr.embedded.JettySolrRunner.start(JettySolrRunner.java:574)
	at org.apache.solr.embedded.JettySolrRunner.start(JettySolrRunner.java:546)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.lambda$createJettys$1(AbstractFullDistribZkTestBase.java:559)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:381)
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
	at java.base/java.lang.Thread.run(Thread.java:1575)
Caused by: java.lang.Exception: Submitter stack trace
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.execute(ExecutorUtil.java:348)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:548)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:408)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1060)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1033)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
	... 1 more

org.apache.solr.client.solrj.jetty.HttpJettySolrClient:org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.client.solrj.jetty.HttpJettySolrClient
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:54)
	at org.apache.solr.client.solrj.jetty.HttpJettySolrClient.<init>(HttpJettySolrClient.java:177)
	at org.apache.solr.client.solrj.jetty.HttpJettySolrClient$Builder.build(HttpJettySolrClient.java:983)
	at org.apache.solr.update.UpdateShardHandler.<init>(UpdateShardHandler.java:121)
	at org.apache.solr.core.CoreContainer.loadInternal(CoreContainer.java:793)
	at org.apache.solr.core.CoreContainer.load(CoreContainer.java:753)
	at org.apache.solr.servlet.CoreContainerProvider.createCoreContainer(CoreContainerProvider.java:343)
	at org.apache.solr.servlet.CoreContainerProvider.init(CoreContainerProvider.java:182)
	at org.apache.solr.servlet.CoreContainerProvider.contextInitialized(CoreContainerProvider.java:83)
	at org.apache.solr.embedded.JettySolrRunner$1.contextInitialized(JettySolrRunner.java:395)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.callContextInitialized(ServletContextHandler.java:1591)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.contextInitialized(ServletContextHandler.java:500)
	at org.eclipse.jetty.ee10.servlet.ServletHandler.initialize(ServletHandler.java:676)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.startContext(ServletContextHandler.java:1325)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.lambda$doStart$0(ServletContextHandler.java:1051)
	at org.eclipse.jetty.server.handler.ContextHandler$ScopedContext.call(ContextHandler.java:1463)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.doStart(ServletContextHandler.java:1048)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:120)
	at org.eclipse.jetty.server.Handler$Abstract.doStart(Handler.java:491)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:120)
	at org.eclipse.jetty.server.Handler$Abstract.doStart(Handler.java:491)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.doStart(GzipHandler.java:132)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:120)
	at org.eclipse.jetty.server.Handler$Abstract.doStart(Handler.java:491)
	at org.eclipse.jetty.server.handler.GracefulHandler.doStart(GracefulHandler.java:134)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
	at org.eclipse.jetty.server.Server.start(Server.java:643)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:120)
	at org.eclipse.jetty.server.Handler$Abstract.doStart(Handler.java:491)
	at org.eclipse.jetty.server.Server.doStart(Server.java:584)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.apache.solr.embedded.JettySolrRunner.retryOnPortBindFailure(JettySolrRunner.java:628)
	at org.apache.solr.embedded.JettySolrRunner.start(JettySolrRunner.java:574)
	at org.apache.solr.embedded.JettySolrRunner.start(JettySolrRunner.java:546)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.lambda$createJettys$1(AbstractFullDistribZkTestBase.java:559)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:381)
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
	at java.base/java.lang.Thread.run(Thread.java:1575)
Caused by: java.lang.Exception: Submitter stack trace
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.execute(ExecutorUtil.java:348)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:548)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:408)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1060)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1033)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
	... 1 more

>

Stack Trace:
java.lang.AssertionError: ObjectTracker found 2 object(s) that were not released!!! [HttpJettySolrClient, HttpJettySolrClient]
org.apache.solr.client.solrj.jetty.HttpJettySolrClient:org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.client.solrj.jetty.HttpJettySolrClient
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:54)
	at org.apache.solr.client.solrj.jetty.HttpJettySolrClient.<init>(HttpJettySolrClient.java:177)
	at org.apache.solr.client.solrj.jetty.HttpJettySolrClient$Builder.build(HttpJettySolrClient.java:983)
	at org.apache.solr.update.UpdateShardHandler.<init>(UpdateShardHandler.java:119)
	at org.apache.solr.core.CoreContainer.loadInternal(CoreContainer.java:793)
	at org.apache.solr.core.CoreContainer.load(CoreContainer.java:753)
	at org.apache.solr.servlet.CoreContainerProvider.createCoreContainer(CoreContainerProvider.java:343)
	at org.apache.solr.servlet.CoreContainerProvider.init(CoreContainerProvider.java:182)
	at org.apache.solr.servlet.CoreContainerProvider.contextInitialized(CoreContainerProvider.java:83)
	at org.apache.solr.embedded.JettySolrRunner$1.contextInitialized(JettySolrRunner.java:395)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.callContextInitialized(ServletContextHandler.java:1591)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.contextInitialized(ServletContextHandler.java:500)
	at org.eclipse.jetty.ee10.servlet.ServletHandler.initialize(ServletHandler.java:676)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.startContext(ServletContextHandler.java:1325)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.lambda$doStart$0(ServletContextHandler.java:1051)
	at org.eclipse.jetty.server.handler.ContextHandler$ScopedContext.call(ContextHandler.java:1463)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.doStart(ServletContextHandler.java:1048)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:120)
	at org.eclipse.jetty.server.Handler$Abstract.doStart(Handler.java:491)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:120)
	at org.eclipse.jetty.server.Handler$Abstract.doStart(Handler.java:491)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.doStart(GzipHandler.java:132)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:120)
	at org.eclipse.jetty.server.Handler$Abstract.doStart(Handler.java:491)
	at org.eclipse.jetty.server.handler.GracefulHandler.doStart(GracefulHandler.java:134)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
	at org.eclipse.jetty.server.Server.start(Server.java:643)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:120)
	at org.eclipse.jetty.server.Handler$Abstract.doStart(Handler.java:491)
	at org.eclipse.jetty.server.Server.doStart(Server.java:584)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.apache.solr.embedded.JettySolrRunner.retryOnPortBindFailure(JettySolrRunner.java:628)
	at org.apache.solr.embedded.JettySolrRunner.start(JettySolrRunner.java:574)
	at org.apache.solr.embedded.JettySolrRunner.start(JettySolrRunner.java:546)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.lambda$createJettys$1(AbstractFullDistribZkTestBase.java:559)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:381)
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
	at java.base/java.lang.Thread.run(Thread.java:1575)
Caused by: java.lang.Exception: Submitter stack trace
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.execute(ExecutorUtil.java:348)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:548)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:408)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1060)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1033)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
	... 1 more

org.apache.solr.client.solrj.jetty.HttpJettySolrClient:org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.client.solrj.jetty.HttpJettySolrClient
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:54)
	at org.apache.solr.client.solrj.jetty.HttpJettySolrClient.<init>(HttpJettySolrClient.java:177)
	at org.apache.solr.client.solrj.jetty.HttpJettySolrClient$Builder.build(HttpJettySolrClient.java:983)
	at org.apache.solr.update.UpdateShardHandler.<init>(UpdateShardHandler.java:121)
	at org.apache.solr.core.CoreContainer.loadInternal(CoreContainer.java:793)
	at org.apache.solr.core.CoreContainer.load(CoreContainer.java:753)
	at org.apache.solr.servlet.CoreContainerProvider.createCoreContainer(CoreContainerProvider.java:343)
	at org.apache.solr.servlet.CoreContainerProvider.init(CoreContainerProvider.java:182)
	at org.apache.solr.servlet.CoreContainerProvider.contextInitialized(CoreContainerProvider.java:83)
	at org.apache.solr.embedded.JettySolrRunner$1.contextInitialized(JettySolrRunner.java:395)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.callContextInitialized(ServletContextHandler.java:1591)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.contextInitialized(ServletContextHandler.java:500)
	at org.eclipse.jetty.ee10.servlet.ServletHandler.initialize(ServletHandler.java:676)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.startContext(ServletContextHandler.java:1325)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.lambda$doStart$0(ServletContextHandler.java:1051)
	at org.eclipse.jetty.server.handler.ContextHandler$ScopedContext.call(ContextHandler.java:1463)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.doStart(ServletContextHandler.java:1048)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:120)
	at org.eclipse.jetty.server.Handler$Abstract.doStart(Handler.java:491)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:120)
	at org.eclipse.jetty.server.Handler$Abstract.doStart(Handler.java:491)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.doStart(GzipHandler.java:132)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:120)
	at org.eclipse.jetty.server.Handler$Abstract.doStart(Handler.java:491)
	at org.eclipse.jetty.server.handler.GracefulHandler.doStart(GracefulHandler.java:134)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
	at org.eclipse.jetty.server.Server.start(Server.java:643)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:120)
	at org.eclipse.jetty.server.Handler$Abstract.doStart(Handler.java:491)
	at org.eclipse.jetty.server.Server.doStart(Server.java:584)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.apache.solr.embedded.JettySolrRunner.retryOnPortBindFailure(JettySolrRunner.java:628)
	at org.apache.solr.embedded.JettySolrRunner.start(JettySolrRunner.java:574)
	at org.apache.solr.embedded.JettySolrRunner.start(JettySolrRunner.java:546)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.lambda$createJettys$1(AbstractFullDistribZkTestBase.java:559)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:381)
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
	at java.base/java.lang.Thread.run(Thread.java:1575)
Caused by: java.lang.Exception: Submitter stack trace
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.execute(ExecutorUtil.java:348)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:548)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:408)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1060)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1033)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
	... 1 more

 expected null, but was:<ObjectTracker found 2 object(s) that were not released!!! [HttpJettySolrClient, HttpJettySolrClient]
org.apache.solr.client.solrj.jetty.HttpJettySolrClient:org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.client.solrj.jetty.HttpJettySolrClient
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:54)
	at org.apache.solr.client.solrj.jetty.HttpJettySolrClient.<init>(HttpJettySolrClient.java:177)
	at org.apache.solr.client.solrj.jetty.HttpJettySolrClient$Builder.build(HttpJettySolrClient.java:983)
	at org.apache.solr.update.UpdateShardHandler.<init>(UpdateShardHandler.java:119)
	at org.apache.solr.core.CoreContainer.loadInternal(CoreContainer.java:793)
	at org.apache.solr.core.CoreContainer.load(CoreContainer.java:753)
	at org.apache.solr.servlet.CoreContainerProvider.createCoreContainer(CoreContainerProvider.java:343)
	at org.apache.solr.servlet.CoreContainerProvider.init(CoreContainerProvider.java:182)
	at org.apache.solr.servlet.CoreContainerProvider.contextInitialized(CoreContainerProvider.java:83)
	at org.apache.solr.embedded.JettySolrRunner$1.contextInitialized(JettySolrRunner.java:395)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.callContextInitialized(ServletContextHandler.java:1591)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.contextInitialized(ServletContextHandler.java:500)
	at org.eclipse.jetty.ee10.servlet.ServletHandler.initialize(ServletHandler.java:676)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.startContext(ServletContextHandler.java:1325)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.lambda$doStart$0(ServletContextHandler.java:1051)
	at org.eclipse.jetty.server.handler.ContextHandler$ScopedContext.call(ContextHandler.java:1463)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.doStart(ServletContextHandler.java:1048)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:120)
	at org.eclipse.jetty.server.Handler$Abstract.doStart(Handler.java:491)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:120)
	at org.eclipse.jetty.server.Handler$Abstract.doStart(Handler.java:491)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.doStart(GzipHandler.java:132)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:120)
	at org.eclipse.jetty.server.Handler$Abstract.doStart(Handler.java:491)
	at org.eclipse.jetty.server.handler.GracefulHandler.doStart(GracefulHandler.java:134)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
	at org.eclipse.jetty.server.Server.start(Server.java:643)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:120)
	at org.eclipse.jetty.server.Handler$Abstract.doStart(Handler.java:491)
	at org.eclipse.jetty.server.Server.doStart(Server.java:584)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.apache.solr.embedded.JettySolrRunner.retryOnPortBindFailure(JettySolrRunner.java:628)
	at org.apache.solr.embedded.JettySolrRunner.start(JettySolrRunner.java:574)
	at org.apache.solr.embedded.JettySolrRunner.start(JettySolrRunner.java:546)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.lambda$createJettys$1(AbstractFullDistribZkTestBase.java:559)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:381)
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
	at java.base/java.lang.Thread.run(Thread.java:1575)
Caused by: java.lang.Exception: Submitter stack trace
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.execute(ExecutorUtil.java:348)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:548)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:408)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1060)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1033)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
	... 1 more

org.apache.solr.client.solrj.jetty.HttpJettySolrClient:org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.client.solrj.jetty.HttpJettySolrClient
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:54)
	at org.apache.solr.client.solrj.jetty.HttpJettySolrClient.<init>(HttpJettySolrClient.java:177)
	at org.apache.solr.client.solrj.jetty.HttpJettySolrClient$Builder.build(HttpJettySolrClient.java:983)
	at org.apache.solr.update.UpdateShardHandler.<init>(UpdateShardHandler.java:121)
	at org.apache.solr.core.CoreContainer.loadInternal(CoreContainer.java:793)
	at org.apache.solr.core.CoreContainer.load(CoreContainer.java:753)
	at org.apache.solr.servlet.CoreContainerProvider.createCoreContainer(CoreContainerProvider.java:343)
	at org.apache.solr.servlet.CoreContainerProvider.init(CoreContainerProvider.java:182)
	at org.apache.solr.servlet.CoreContainerProvider.contextInitialized(CoreContainerProvider.java:83)
	at org.apache.solr.embedded.JettySolrRunner$1.contextInitialized(JettySolrRunner.java:395)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.callContextInitialized(ServletContextHandler.java:1591)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.contextInitialized(ServletContextHandler.java:500)
	at org.eclipse.jetty.ee10.servlet.ServletHandler.initialize(ServletHandler.java:676)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.startContext(ServletContextHandler.java:1325)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.lambda$doStart$0(ServletContextHandler.java:1051)
	at org.eclipse.jetty.server.handler.ContextHandler$ScopedContext.call(ContextHandler.java:1463)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.doStart(ServletContextHandler.java:1048)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:120)
	at org.eclipse.jetty.server.Handler$Abstract.doStart(Handler.java:491)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:120)
	at org.eclipse.jetty.server.Handler$Abstract.doStart(Handler.java:491)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.doStart(GzipHandler.java:132)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:120)
	at org.eclipse.jetty.server.Handler$Abstract.doStart(Handler.java:491)
	at org.eclipse.jetty.server.handler.GracefulHandler.doStart(GracefulHandler.java:134)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
	at org.eclipse.jetty.server.Server.start(Server.java:643)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:120)
	at org.eclipse.jetty.server.Handler$Abstract.doStart(Handler.java:491)
	at org.eclipse.jetty.server.Server.doStart(Server.java:584)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.apache.solr.embedded.JettySolrRunner.retryOnPortBindFailure(JettySolrRunner.java:628)
	at org.apache.solr.embedded.JettySolrRunner.start(JettySolrRunner.java:574)
	at org.apache.solr.embedded.JettySolrRunner.start(JettySolrRunner.java:546)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.lambda$createJettys$1(AbstractFullDistribZkTestBase.java:559)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:381)
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
	at java.base/java.lang.Thread.run(Thread.java:1575)
Caused by: java.lang.Exception: Submitter stack trace
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.execute(ExecutorUtil.java:348)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:548)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:408)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1060)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1033)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
	... 1 more

>
	at org.junit.Assert.fail(Assert.java:89)
	at org.junit.Assert.failNotNull(Assert.java:756)
	at org.junit.Assert.assertNull(Assert.java:738)
	at org.apache.solr.SolrTestCase$1.afterIfSuccessful(SolrTestCase.java:101)
	at com.carrotsearch.randomizedtesting.rules.TestRuleAdapter$1.afterIfSuccessful(TestRuleAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:37)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
	at java.base/java.lang.Thread.run(Thread.java:1575)


FAILED:  org.apache.solr.cloud.BasicDistributedZkTest.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 9 threads leaked from SUITE scope at org.apache.solr.cloud.BasicDistributedZkTest: 
   1) Thread[id=1229, name=h2sc-488-thread-7, state=WAITING, group=TGRP-BasicDistributedZkTest]
        at java.base/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
        at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
        at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
        at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
        at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
        at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
        at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
        at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
        at java.base/java.lang.Thread.run(Thread.java:1575)
   2) Thread[id=1221, name=h2sc-488-thread-6, state=WAITING, group=TGRP-BasicDistributedZkTest]
        at java.base/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
        at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
        at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
        at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
        at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
        at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
        at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
        at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
        at java.base/java.lang.Thread.run(Thread.java:1575)
   3) Thread[id=733, name=h2sc-488-thread-1, state=RUNNABLE, group=TGRP-BasicDistributedZkTest]
        at java.base/sun.nio.ch.EPoll.wait(Native Method)
        at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
        at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
        at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
        at org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:182)
        at org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:189)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:614)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:548)
        at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produceTask(AdaptiveExecutionStrategy.java:514)
        at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:258)
        at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produce(AdaptiveExecutionStrategy.java:195)
        at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:381)
        at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
        at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
        at java.base/java.lang.Thread.run(Thread.java:1575)
   4) Thread[id=732, name=h2sc-487-thread-2, state=RUNNABLE, group=TGRP-BasicDistributedZkTest]
        at java.base/sun.nio.ch.EPoll.wait(Native Method)
        at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
        at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
        at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
        at org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:182)
        at org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:189)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:614)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:548)
        at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produceTask(AdaptiveExecutionStrategy.java:514)
        at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:258)
        at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produce(AdaptiveExecutionStrategy.java:195)
        at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:381)
        at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
        at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
        at java.base/java.lang.Thread.run(Thread.java:1575)
   5) Thread[id=734, name=h2sc-488-thread-2, state=RUNNABLE, group=TGRP-BasicDistributedZkTest]
        at java.base/sun.nio.ch.EPoll.wait(Native Method)
        at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
        at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
        at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
        at org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:182)
        at org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:189)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:614)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:548)
        at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produceTask(AdaptiveExecutionStrategy.java:514)
        at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:258)
        at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produce(AdaptiveExecutionStrategy.java:195)
        at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:381)
        at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
        at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
        at java.base/java.lang.Thread.run(Thread.java:1575)
   6) Thread[id=1050, name=h2sc-488-thread-5, state=WAITING, group=TGRP-BasicDistributedZkTest]
        at java.base/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
        at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
        at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
        at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
        at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
        at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
        at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
        at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
        at java.base/java.lang.Thread.run(Thread.java:1575)
   7) Thread[id=924, name=h2sc-488-thread-3, state=WAITING, group=TGRP-BasicDistributedZkTest]
        at java.base/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
        at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
        at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
        at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
        at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
        at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
        at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
        at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
        at java.base/java.lang.Thread.run(Thread.java:1575)
   8) Thread[id=925, name=HttpClient@5571a67a-scheduler-1, state=WAITING, group=TGRP-BasicDistributedZkTest]
        at java.base/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
        at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
        at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
        at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
        at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
        at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
        at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
        at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
        at java.base/java.lang.Thread.run(Thread.java:1575)
   9) Thread[id=926, name=h2sc-488-thread-4, state=WAITING, group=TGRP-BasicDistributedZkTest]
        at java.base/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
        at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
        at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
        at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
        at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
        at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
        at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
        at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
        at java.base/java.lang.Thread.run(Thread.java:1575)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 9 threads leaked from SUITE scope at org.apache.solr.cloud.BasicDistributedZkTest: 
   1) Thread[id=1229, name=h2sc-488-thread-7, state=WAITING, group=TGRP-BasicDistributedZkTest]
        at java.base/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
        at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
        at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
        at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
        at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
        at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
        at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
        at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
        at java.base/java.lang.Thread.run(Thread.java:1575)
   2) Thread[id=1221, name=h2sc-488-thread-6, state=WAITING, group=TGRP-BasicDistributedZkTest]
        at java.base/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
        at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
        at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
        at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
        at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
        at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
        at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
        at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
        at java.base/java.lang.Thread.run(Thread.java:1575)
   3) Thread[id=733, name=h2sc-488-thread-1, state=RUNNABLE, group=TGRP-BasicDistributedZkTest]
        at java.base/sun.nio.ch.EPoll.wait(Native Method)
        at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
        at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
        at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
        at org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:182)
        at org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:189)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:614)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:548)
        at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produceTask(AdaptiveExecutionStrategy.java:514)
        at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:258)
        at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produce(AdaptiveExecutionStrategy.java:195)
        at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:381)
        at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
        at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
        at java.base/java.lang.Thread.run(Thread.java:1575)
   4) Thread[id=732, name=h2sc-487-thread-2, state=RUNNABLE, group=TGRP-BasicDistributedZkTest]
        at java.base/sun.nio.ch.EPoll.wait(Native Method)
        at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
        at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
        at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
        at org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:182)
        at org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:189)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:614)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:548)
        at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produceTask(AdaptiveExecutionStrategy.java:514)
        at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:258)
        at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produce(AdaptiveExecutionStrategy.java:195)
        at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:381)
        at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
        at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
        at java.base/java.lang.Thread.run(Thread.java:1575)
   5) Thread[id=734, name=h2sc-488-thread-2, state=RUNNABLE, group=TGRP-BasicDistributedZkTest]
        at java.base/sun.nio.ch.EPoll.wait(Native Method)
        at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
        at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
        at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
        at org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:182)
        at org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:189)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:614)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:548)
        at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produceTask(AdaptiveExecutionStrategy.java:514)
        at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:258)
        at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produce(AdaptiveExecutionStrategy.java:195)
        at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:381)
        at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
        at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
        at java.base/java.lang.Thread.run(Thread.java:1575)
   6) Thread[id=1050, name=h2sc-488-thread-5, state=WAITING, group=TGRP-BasicDistributedZkTest]
        at java.base/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
        at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
        at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
        at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
        at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
        at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
        at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
        at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
        at java.base/java.lang.Thread.run(Thread.java:1575)
   7) Thread[id=924, name=h2sc-488-thread-3, state=WAITING, group=TGRP-BasicDistributedZkTest]
        at java.base/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
        at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
        at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
        at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
        at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
        at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
        at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
        at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
        at java.base/java.lang.Thread.run(Thread.java:1575)
   8) Thread[id=925, name=HttpClient@5571a67a-scheduler-1, state=WAITING, group=TGRP-BasicDistributedZkTest]
        at java.base/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
        at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
        at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
        at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
        at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
        at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
        at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
        at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
        at java.base/java.lang.Thread.run(Thread.java:1575)
   9) Thread[id=926, name=h2sc-488-thread-4, state=WAITING, group=TGRP-BasicDistributedZkTest]
        at java.base/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
        at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
        at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
        at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
        at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
        at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
        at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
        at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
        at java.base/java.lang.Thread.run(Thread.java:1575)
	at __randomizedtesting.SeedInfo.seed([8BC7FBC52FF5D780]:0)


FAILED:  org.apache.solr.crossdc.manager.SolrAndKafkaIntegrationTest.testStrictOrdering

Error Message:
java.lang.AssertionError: results={responseHeader={zkConnected=true, status=0, QTime=0, params={q=*:*, _stateVer_=collection1:3, wt=javabin}}, response={numFound=1,numFoundExact=true,start=0,docs=[SolrDocument{id=id-1, _version_=1859091910854967297, _root_=id-1}]}}

Stack Trace:
java.lang.AssertionError: results={responseHeader={zkConnected=true, status=0, QTime=0, params={q=*:*, _stateVer_=collection1:3, wt=javabin}}, response={numFound=1,numFoundExact=true,start=0,docs=[SolrDocument{id=id-1, _version_=1859091910854967297, _root_=id-1}]}}
	at __randomizedtesting.SeedInfo.seed([8BC7FBC52FF5D780:A27A243C096D7CE1]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.junit.Assert.assertTrue(Assert.java:42)
	at org.apache.solr.crossdc.manager.SolrAndKafkaIntegrationTest.assertClusterEventuallyHasDocs(SolrAndKafkaIntegrationTest.java:464)
	at org.apache.solr.crossdc.manager.SolrAndKafkaIntegrationTest.assertCluster2EventuallyHasDocs(SolrAndKafkaIntegrationTest.java:446)
	at org.apache.solr.crossdc.manager.SolrAndKafkaIntegrationTest.testStrictOrdering(SolrAndKafkaIntegrationTest.java:267)
	at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103)
	at java.base/java.lang.reflect.Method.invoke(Method.java:580)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1763)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
	at java.base/java.lang.Thread.run(Thread.java:1575)

[JENKINS] Solr-main-Linux (64bit/hotspot/jdk-22.0.2) - Build # 32038 - Still Unstable!

Posted by Policeman Jenkins Server via builds <bu...@solr.apache.org>.
Build: https://jenkins.thetaphi.de/job/Solr-main-Linux/32038/
Java: 64bit/hotspot/jdk-22.0.2 -XX:-UseCompressedOops -XX:+UseSerialGC

5 tests failed.
FAILED:  org.apache.solr.cloud.api.collections.CollectionsAPIDistributedZkTest.deletePartiallyCreatedCollection

Error Message:
org.apache.solr.client.solrj.RemoteSolrException: Error from server at http://127.0.0.1:39933/solr: org.apache.solr.common.SolrException: Underlying core creation failed while creating collection: halfdeletedcollection

Stack Trace:
org.apache.solr.client.solrj.RemoteSolrException: Error from server at http://127.0.0.1:39933/solr: org.apache.solr.common.SolrException: Underlying core creation failed while creating collection: halfdeletedcollection
	at __randomizedtesting.SeedInfo.seed([C896D7FD3481FFBE:E216BB18835BDF76]:0)
	at app//org.apache.solr.client.solrj.apache.HttpSolrClient.executeMethod(HttpSolrClient.java:686)
	at app//org.apache.solr.client.solrj.apache.HttpSolrClient.request(HttpSolrClient.java:256)
	at app//org.apache.solr.client.solrj.apache.HttpSolrClient.request(HttpSolrClient.java:235)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:629)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:613)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:638)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:582)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1570)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1205)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:1127)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:302)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:348)
	at app//org.apache.solr.cloud.api.collections.AbstractCollectionsAPIDistributedZkTestBase.deletePartiallyCreatedCollection(AbstractCollectionsAPIDistributedZkTestBase.java:151)
	at java.base@22.0.2/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103)
	at java.base@22.0.2/java.lang.reflect.Method.invoke(Method.java:580)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1763)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
	at java.base@22.0.2/java.lang.Thread.run(Thread.java:1570)


FAILED:  org.apache.solr.cloud.api.collections.CollectionsAPIDistributedZkTest.testDeleteNonExistentCollection

Error Message:
org.apache.solr.client.solrj.RemoteSolrException: Error from server at http://127.0.0.1:39933/solr: org.apache.solr.common.SolrException: Underlying core creation failed while creating collection: acollectionafterbaddelete

Stack Trace:
org.apache.solr.client.solrj.RemoteSolrException: Error from server at http://127.0.0.1:39933/solr: org.apache.solr.common.SolrException: Underlying core creation failed while creating collection: acollectionafterbaddelete
	at __randomizedtesting.SeedInfo.seed([C896D7FD3481FFBE:2FB1C7C023E28047]:0)
	at app//org.apache.solr.client.solrj.apache.HttpSolrClient.executeMethod(HttpSolrClient.java:686)
	at app//org.apache.solr.client.solrj.apache.HttpSolrClient.request(HttpSolrClient.java:256)
	at app//org.apache.solr.client.solrj.apache.HttpSolrClient.request(HttpSolrClient.java:235)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:629)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:613)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:638)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:582)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1570)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1205)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:1127)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:302)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:348)
	at app//org.apache.solr.cloud.api.collections.AbstractCollectionsAPIDistributedZkTestBase.testDeleteNonExistentCollection(AbstractCollectionsAPIDistributedZkTestBase.java:334)
	at java.base@22.0.2/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103)
	at java.base@22.0.2/java.lang.reflect.Method.invoke(Method.java:580)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1763)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
	at java.base@22.0.2/java.lang.Thread.run(Thread.java:1570)


FAILED:  org.apache.solr.cloud.api.collections.CollectionsAPIDistributedZkTest.testSpecificConfigsets

Error Message:
org.apache.solr.client.solrj.RemoteSolrException: Error from server at http://127.0.0.1:37001/solr: org.apache.solr.common.SolrException: Underlying core creation failed while creating collection: withconfigset2

Stack Trace:
org.apache.solr.client.solrj.RemoteSolrException: Error from server at http://127.0.0.1:37001/solr: org.apache.solr.common.SolrException: Underlying core creation failed while creating collection: withconfigset2
	at __randomizedtesting.SeedInfo.seed([C896D7FD3481FFBE:E5E898A7C3A545B2]:0)
	at app//org.apache.solr.client.solrj.apache.HttpSolrClient.executeMethod(HttpSolrClient.java:686)
	at app//org.apache.solr.client.solrj.apache.HttpSolrClient.request(HttpSolrClient.java:256)
	at app//org.apache.solr.client.solrj.apache.HttpSolrClient.request(HttpSolrClient.java:235)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:629)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:613)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:638)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:582)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1570)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1205)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:1127)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:302)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:348)
	at app//org.apache.solr.cloud.api.collections.AbstractCollectionsAPIDistributedZkTestBase.testSpecificConfigsets(AbstractCollectionsAPIDistributedZkTestBase.java:344)
	at java.base@22.0.2/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103)
	at java.base@22.0.2/java.lang.reflect.Method.invoke(Method.java:580)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1763)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
	at java.base@22.0.2/java.lang.Thread.run(Thread.java:1570)


FAILED:  org.apache.solr.cloud.api.collections.CollectionsAPIDistributedZkTest.testSpecificConfigsets

Error Message:
org.apache.solr.client.solrj.RemoteSolrException: Error from server at http://127.0.0.1:39933/solr: org.apache.solr.common.SolrException: Underlying core creation failed while creating collection: withconfigset2

Stack Trace:
org.apache.solr.client.solrj.RemoteSolrException: Error from server at http://127.0.0.1:39933/solr: org.apache.solr.common.SolrException: Underlying core creation failed while creating collection: withconfigset2
	at __randomizedtesting.SeedInfo.seed([C896D7FD3481FFBE:E5E898A7C3A545B2]:0)
	at app//org.apache.solr.client.solrj.apache.HttpSolrClient.executeMethod(HttpSolrClient.java:686)
	at app//org.apache.solr.client.solrj.apache.HttpSolrClient.request(HttpSolrClient.java:256)
	at app//org.apache.solr.client.solrj.apache.HttpSolrClient.request(HttpSolrClient.java:235)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:629)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:613)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:638)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:582)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1570)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1205)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:1127)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:302)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:348)
	at app//org.apache.solr.cloud.api.collections.AbstractCollectionsAPIDistributedZkTestBase.testSpecificConfigsets(AbstractCollectionsAPIDistributedZkTestBase.java:344)
	at java.base@22.0.2/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103)
	at java.base@22.0.2/java.lang.reflect.Method.invoke(Method.java:580)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1763)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
	at java.base@22.0.2/java.lang.Thread.run(Thread.java:1570)


FAILED:  org.apache.solr.crossdc.manager.SolrAndKafkaIntegrationTest.testStrictOrdering

Error Message:
java.lang.AssertionError: results={responseHeader={zkConnected=true, status=0, QTime=0, params={q=*:*, _stateVer_=collection1:3, wt=javabin}}, response={numFound=2,numFoundExact=true,start=0,docs=[SolrDocument{id=id-0, _version_=1859097855433637888, _root_=id-0}, SolrDocument{id=id-1, _version_=1859097855433637889, _root_=id-1}]}}

Stack Trace:
java.lang.AssertionError: results={responseHeader={zkConnected=true, status=0, QTime=0, params={q=*:*, _stateVer_=collection1:3, wt=javabin}}, response={numFound=2,numFoundExact=true,start=0,docs=[SolrDocument{id=id-0, _version_=1859097855433637888, _root_=id-0}, SolrDocument{id=id-1, _version_=1859097855433637889, _root_=id-1}]}}
	at __randomizedtesting.SeedInfo.seed([C896D7FD3481FFBE:E12B0804121954DF]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.junit.Assert.assertTrue(Assert.java:42)
	at org.apache.solr.crossdc.manager.SolrAndKafkaIntegrationTest.assertClusterEventuallyHasDocs(SolrAndKafkaIntegrationTest.java:464)
	at org.apache.solr.crossdc.manager.SolrAndKafkaIntegrationTest.assertCluster2EventuallyHasDocs(SolrAndKafkaIntegrationTest.java:446)
	at org.apache.solr.crossdc.manager.SolrAndKafkaIntegrationTest.testStrictOrdering(SolrAndKafkaIntegrationTest.java:267)
	at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103)
	at java.base/java.lang.reflect.Method.invoke(Method.java:580)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1763)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
	at java.base/java.lang.Thread.run(Thread.java:1570)

[JENKINS] Solr-main-Linux (64bit/hotspot/jdk-21.0.6) - Build # 32037 - Still Unstable!

Posted by Policeman Jenkins Server via builds <bu...@solr.apache.org>.
Build: https://jenkins.thetaphi.de/job/Solr-main-Linux/32037/
Java: 64bit/hotspot/jdk-21.0.6 -XX:+UseCompressedOops -XX:+UseSerialGC

1 tests failed.
FAILED:  org.apache.solr.cloud.DeleteNodeTest.classMethod

Error Message:
java.lang.AssertionError: ObjectTracker found 2 object(s) that were not released!!! [ZkShardTerms, ZkShardTerms]
org.apache.solr.cloud.ZkShardTerms:org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.cloud.ZkShardTerms
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:54)
	at org.apache.solr.cloud.ZkShardTerms.<init>(ZkShardTerms.java:106)
	at org.apache.solr.cloud.ZkCollectionTerms.lambda$getShard$0(ZkCollectionTerms.java:45)
	at java.base/java.util.HashMap.compute(HashMap.java:1324)
	at org.apache.solr.cloud.ZkCollectionTerms.getShard(ZkCollectionTerms.java:41)
	at org.apache.solr.cloud.ZkController.getShardTerms(ZkController.java:1939)
	at org.apache.solr.cloud.ZkController.publish(ZkController.java:1889)
	at org.apache.solr.cloud.ZkController.publish(ZkController.java:1805)
	at org.apache.solr.cloud.ShardLeaderElectionContext.publishActiveIfRegisteredAndNotActive(ShardLeaderElectionContext.java:428)
	at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ShardLeaderElectionContext.java:302)
	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:168)
	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:129)
	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:342)
	at org.apache.solr.common.cloud.SolrZkClient$ProcessWatchWithExecutor.lambda$process$0(SolrZkClient.java:1061)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:381)
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
	at java.base/java.lang.Thread.run(Thread.java:1583)
Caused by: java.lang.Exception: Submitter stack trace
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.execute(ExecutorUtil.java:348)
	at org.apache.solr.common.cloud.SolrZkClient$ProcessWatchWithExecutor.process(SolrZkClient.java:1057)
	at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:69)
	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:566)
	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:541)

org.apache.solr.cloud.ZkShardTerms:org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.cloud.ZkShardTerms
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:54)
	at org.apache.solr.cloud.ZkShardTerms.<init>(ZkShardTerms.java:106)
	at org.apache.solr.cloud.ZkCollectionTerms.lambda$getShard$0(ZkCollectionTerms.java:45)
	at java.base/java.util.HashMap.compute(HashMap.java:1324)
	at org.apache.solr.cloud.ZkCollectionTerms.getShard(ZkCollectionTerms.java:41)
	at org.apache.solr.cloud.ZkController.getShardTerms(ZkController.java:1939)
	at org.apache.solr.cloud.ZkController.publish(ZkController.java:1889)
	at org.apache.solr.cloud.ZkController.publish(ZkController.java:1805)
	at org.apache.solr.cloud.ShardLeaderElectionContext.publishActiveIfRegisteredAndNotActive(ShardLeaderElectionContext.java:428)
	at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ShardLeaderElectionContext.java:302)
	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:168)
	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:129)
	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:342)
	at org.apache.solr.common.cloud.SolrZkClient$ProcessWatchWithExecutor.lambda$process$0(SolrZkClient.java:1061)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:381)
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
	at java.base/java.lang.Thread.run(Thread.java:1583)
Caused by: java.lang.Exception: Submitter stack trace
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.execute(ExecutorUtil.java:348)
	at org.apache.solr.common.cloud.SolrZkClient$ProcessWatchWithExecutor.process(SolrZkClient.java:1057)
	at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:69)
	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:566)
	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:541)

 expected null, but was:<ObjectTracker found 2 object(s) that were not released!!! [ZkShardTerms, ZkShardTerms]
org.apache.solr.cloud.ZkShardTerms:org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.cloud.ZkShardTerms
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:54)
	at org.apache.solr.cloud.ZkShardTerms.<init>(ZkShardTerms.java:106)
	at org.apache.solr.cloud.ZkCollectionTerms.lambda$getShard$0(ZkCollectionTerms.java:45)
	at java.base/java.util.HashMap.compute(HashMap.java:1324)
	at org.apache.solr.cloud.ZkCollectionTerms.getShard(ZkCollectionTerms.java:41)
	at org.apache.solr.cloud.ZkController.getShardTerms(ZkController.java:1939)
	at org.apache.solr.cloud.ZkController.publish(ZkController.java:1889)
	at org.apache.solr.cloud.ZkController.publish(ZkController.java:1805)
	at org.apache.solr.cloud.ShardLeaderElectionContext.publishActiveIfRegisteredAndNotActive(ShardLeaderElectionContext.java:428)
	at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ShardLeaderElectionContext.java:302)
	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:168)
	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:129)
	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:342)
	at org.apache.solr.common.cloud.SolrZkClient$ProcessWatchWithExecutor.lambda$process$0(SolrZkClient.java:1061)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:381)
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
	at java.base/java.lang.Thread.run(Thread.java:1583)
Caused by: java.lang.Exception: Submitter stack trace
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.execute(ExecutorUtil.java:348)
	at org.apache.solr.common.cloud.SolrZkClient$ProcessWatchWithExecutor.process(SolrZkClient.java:1057)
	at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:69)
	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:566)
	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:541)

org.apache.solr.cloud.ZkShardTerms:org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.cloud.ZkShardTerms
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:54)
	at org.apache.solr.cloud.ZkShardTerms.<init>(ZkShardTerms.java:106)
	at org.apache.solr.cloud.ZkCollectionTerms.lambda$getShard$0(ZkCollectionTerms.java:45)
	at java.base/java.util.HashMap.compute(HashMap.java:1324)
	at org.apache.solr.cloud.ZkCollectionTerms.getShard(ZkCollectionTerms.java:41)
	at org.apache.solr.cloud.ZkController.getShardTerms(ZkController.java:1939)
	at org.apache.solr.cloud.ZkController.publish(ZkController.java:1889)
	at org.apache.solr.cloud.ZkController.publish(ZkController.java:1805)
	at org.apache.solr.cloud.ShardLeaderElectionContext.publishActiveIfRegisteredAndNotActive(ShardLeaderElectionContext.java:428)
	at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ShardLeaderElectionContext.java:302)
	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:168)
	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:129)
	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:342)
	at org.apache.solr.common.cloud.SolrZkClient$ProcessWatchWithExecutor.lambda$process$0(SolrZkClient.java:1061)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:381)
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
	at java.base/java.lang.Thread.run(Thread.java:1583)
Caused by: java.lang.Exception: Submitter stack trace
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.execute(ExecutorUtil.java:348)
	at org.apache.solr.common.cloud.SolrZkClient$ProcessWatchWithExecutor.process(SolrZkClient.java:1057)
	at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:69)
	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:566)
	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:541)

>

Stack Trace:
java.lang.AssertionError: ObjectTracker found 2 object(s) that were not released!!! [ZkShardTerms, ZkShardTerms]
org.apache.solr.cloud.ZkShardTerms:org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.cloud.ZkShardTerms
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:54)
	at org.apache.solr.cloud.ZkShardTerms.<init>(ZkShardTerms.java:106)
	at org.apache.solr.cloud.ZkCollectionTerms.lambda$getShard$0(ZkCollectionTerms.java:45)
	at java.base/java.util.HashMap.compute(HashMap.java:1324)
	at org.apache.solr.cloud.ZkCollectionTerms.getShard(ZkCollectionTerms.java:41)
	at org.apache.solr.cloud.ZkController.getShardTerms(ZkController.java:1939)
	at org.apache.solr.cloud.ZkController.publish(ZkController.java:1889)
	at org.apache.solr.cloud.ZkController.publish(ZkController.java:1805)
	at org.apache.solr.cloud.ShardLeaderElectionContext.publishActiveIfRegisteredAndNotActive(ShardLeaderElectionContext.java:428)
	at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ShardLeaderElectionContext.java:302)
	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:168)
	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:129)
	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:342)
	at org.apache.solr.common.cloud.SolrZkClient$ProcessWatchWithExecutor.lambda$process$0(SolrZkClient.java:1061)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:381)
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
	at java.base/java.lang.Thread.run(Thread.java:1583)
Caused by: java.lang.Exception: Submitter stack trace
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.execute(ExecutorUtil.java:348)
	at org.apache.solr.common.cloud.SolrZkClient$ProcessWatchWithExecutor.process(SolrZkClient.java:1057)
	at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:69)
	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:566)
	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:541)

org.apache.solr.cloud.ZkShardTerms:org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.cloud.ZkShardTerms
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:54)
	at org.apache.solr.cloud.ZkShardTerms.<init>(ZkShardTerms.java:106)
	at org.apache.solr.cloud.ZkCollectionTerms.lambda$getShard$0(ZkCollectionTerms.java:45)
	at java.base/java.util.HashMap.compute(HashMap.java:1324)
	at org.apache.solr.cloud.ZkCollectionTerms.getShard(ZkCollectionTerms.java:41)
	at org.apache.solr.cloud.ZkController.getShardTerms(ZkController.java:1939)
	at org.apache.solr.cloud.ZkController.publish(ZkController.java:1889)
	at org.apache.solr.cloud.ZkController.publish(ZkController.java:1805)
	at org.apache.solr.cloud.ShardLeaderElectionContext.publishActiveIfRegisteredAndNotActive(ShardLeaderElectionContext.java:428)
	at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ShardLeaderElectionContext.java:302)
	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:168)
	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:129)
	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:342)
	at org.apache.solr.common.cloud.SolrZkClient$ProcessWatchWithExecutor.lambda$process$0(SolrZkClient.java:1061)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:381)
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
	at java.base/java.lang.Thread.run(Thread.java:1583)
Caused by: java.lang.Exception: Submitter stack trace
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.execute(ExecutorUtil.java:348)
	at org.apache.solr.common.cloud.SolrZkClient$ProcessWatchWithExecutor.process(SolrZkClient.java:1057)
	at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:69)
	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:566)
	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:541)

 expected null, but was:<ObjectTracker found 2 object(s) that were not released!!! [ZkShardTerms, ZkShardTerms]
org.apache.solr.cloud.ZkShardTerms:org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.cloud.ZkShardTerms
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:54)
	at org.apache.solr.cloud.ZkShardTerms.<init>(ZkShardTerms.java:106)
	at org.apache.solr.cloud.ZkCollectionTerms.lambda$getShard$0(ZkCollectionTerms.java:45)
	at java.base/java.util.HashMap.compute(HashMap.java:1324)
	at org.apache.solr.cloud.ZkCollectionTerms.getShard(ZkCollectionTerms.java:41)
	at org.apache.solr.cloud.ZkController.getShardTerms(ZkController.java:1939)
	at org.apache.solr.cloud.ZkController.publish(ZkController.java:1889)
	at org.apache.solr.cloud.ZkController.publish(ZkController.java:1805)
	at org.apache.solr.cloud.ShardLeaderElectionContext.publishActiveIfRegisteredAndNotActive(ShardLeaderElectionContext.java:428)
	at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ShardLeaderElectionContext.java:302)
	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:168)
	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:129)
	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:342)
	at org.apache.solr.common.cloud.SolrZkClient$ProcessWatchWithExecutor.lambda$process$0(SolrZkClient.java:1061)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:381)
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
	at java.base/java.lang.Thread.run(Thread.java:1583)
Caused by: java.lang.Exception: Submitter stack trace
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.execute(ExecutorUtil.java:348)
	at org.apache.solr.common.cloud.SolrZkClient$ProcessWatchWithExecutor.process(SolrZkClient.java:1057)
	at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:69)
	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:566)
	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:541)

org.apache.solr.cloud.ZkShardTerms:org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.cloud.ZkShardTerms
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:54)
	at org.apache.solr.cloud.ZkShardTerms.<init>(ZkShardTerms.java:106)
	at org.apache.solr.cloud.ZkCollectionTerms.lambda$getShard$0(ZkCollectionTerms.java:45)
	at java.base/java.util.HashMap.compute(HashMap.java:1324)
	at org.apache.solr.cloud.ZkCollectionTerms.getShard(ZkCollectionTerms.java:41)
	at org.apache.solr.cloud.ZkController.getShardTerms(ZkController.java:1939)
	at org.apache.solr.cloud.ZkController.publish(ZkController.java:1889)
	at org.apache.solr.cloud.ZkController.publish(ZkController.java:1805)
	at org.apache.solr.cloud.ShardLeaderElectionContext.publishActiveIfRegisteredAndNotActive(ShardLeaderElectionContext.java:428)
	at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ShardLeaderElectionContext.java:302)
	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:168)
	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:129)
	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:342)
	at org.apache.solr.common.cloud.SolrZkClient$ProcessWatchWithExecutor.lambda$process$0(SolrZkClient.java:1061)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:381)
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
	at java.base/java.lang.Thread.run(Thread.java:1583)
Caused by: java.lang.Exception: Submitter stack trace
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.execute(ExecutorUtil.java:348)
	at org.apache.solr.common.cloud.SolrZkClient$ProcessWatchWithExecutor.process(SolrZkClient.java:1057)
	at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:69)
	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:566)
	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:541)

>
	at __randomizedtesting.SeedInfo.seed([EE1F40A70031CD63]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.junit.Assert.failNotNull(Assert.java:756)
	at org.junit.Assert.assertNull(Assert.java:738)
	at org.apache.solr.SolrTestCase$1.afterIfSuccessful(SolrTestCase.java:101)
	at com.carrotsearch.randomizedtesting.rules.TestRuleAdapter$1.afterIfSuccessful(TestRuleAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:37)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
	at java.base/java.lang.Thread.run(Thread.java:1583)