You are viewing a plain text version of this content. The canonical link for it is here.
Posted to builds@solr.apache.org by Policeman Jenkins Server via builds <bu...@solr.apache.org> on 2026/05/18 01:12:51 UTC

[JENKINS-EA] Solr-main-Linux (64bit/hotspot/jdk-27-ea+14) - Build # 33377 - Still Unstable!

Build: https://jenkins.thetaphi.de/job/Solr-main-Linux/33377/
Java: 64bit/hotspot/jdk-27-ea+14 -XX:+UseCompressedOops -XX:+UseZGC

1 tests failed.
FAILED:  org.apache.solr.crossdc.manager.consumer.KafkaCrossDcConsumerTest.testSolrClientSupplier

Error Message:
java.lang.OutOfMemoryError: Java heap space

Stack Trace:
java.lang.OutOfMemoryError: Java heap space
	at java.base/jdk.internal.loader.URLClassPath$FileLoader.getResource(URLClassPath.java:942)
	at java.base/jdk.internal.loader.URLClassPath.getResource(URLClassPath.java:324)
	at java.base/jdk.internal.loader.BuiltinClassLoader.findClassOnClassPathOrNull(BuiltinClassLoader.java:688)
	at java.base/jdk.internal.loader.BuiltinClassLoader.loadClassOrNull(BuiltinClassLoader.java:620)
	at java.base/jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:578)
	at java.base/java.lang.ClassLoader.loadClass(ClassLoader.java:502)
	at org.mockito.internal.creation.bytebuddy.MockMethodAdvice.handle(MockMethodAdvice.java:138)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.close(CloudSolrClient.java:667)
	at org.apache.solr.common.util.IOUtils.closeQuietly(IOUtils.java:34)
	at org.apache.solr.crossdc.manager.consumer.KafkaCrossDcConsumer$SolrClientSupplier.get(KafkaCrossDcConsumer.java:157)
	at org.apache.solr.crossdc.manager.consumer.KafkaCrossDcConsumerTest.testSolrClientSupplier(KafkaCrossDcConsumerTest.java:211)
	at java.base/java.lang.invoke.LambdaForm$DMH/0x000000001e124800.invokeVirtual(LambdaForm$DMH)
	at java.base/java.lang.invoke.LambdaForm$MH/0x000000001e3a4000.invoke(LambdaForm$MH)
	at java.base/java.lang.invoke.Invokers$Holder.invokeExact_MT(Invokers$Holder)
	at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invokeImpl(DirectMethodHandleAccessor.java:154)
	at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
	at java.base/java.lang.reflect.Method.invoke(Method.java:565)
	at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
	at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
	at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
	at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
	at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
	at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
	at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
	at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
	at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
	at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
	at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)

[JENKINS] Solr-main-Linux (64bit/hotspot/jdk-22.0.2) - Build # 33381 - Still Unstable!

Posted by Policeman Jenkins Server via builds <bu...@solr.apache.org>.
Build: https://jenkins.thetaphi.de/job/Solr-main-Linux/33381/
Java: 64bit/hotspot/jdk-22.0.2 -XX:+UseCompressedOops -XX:+UseZGC

2 tests failed.
FAILED:  org.apache.solr.handler.component.AsyncTrackerSemaphoreLeakTest.testSemaphoreLeakOnLBRetry

Error Message:
java.lang.AssertionError: BUG (LBJettySolrClient retry deadlock): futures did not complete within 30s. IO threads are permanently blocked in semaphore.acquire() because the retry fires synchronously on the IO thread before onComplete can release().

Stack Trace:
java.lang.AssertionError: BUG (LBJettySolrClient retry deadlock): futures did not complete within 30s. IO threads are permanently blocked in semaphore.acquire() because the retry fires synchronously on the IO thread before onComplete can release().
	at __randomizedtesting.SeedInfo.seed([EEC8FBE27CD386FB:3D68000916BBDE79]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.solr.handler.component.AsyncTrackerSemaphoreLeakTest.testSemaphoreLeakOnLBRetry(AsyncTrackerSemaphoreLeakTest.java:201)
	at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103)
	at java.base/java.lang.reflect.Method.invoke(Method.java:580)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1763)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
	at java.base/java.lang.Thread.run(Thread.java:1570)


FAILED:  org.apache.solr.handler.component.AsyncTrackerSemaphoreLeakTest.testSemaphoreLeakOnLBRetry

Error Message:
java.lang.AssertionError: BUG (LBJettySolrClient retry deadlock): futures did not complete within 30s. IO threads are permanently blocked in semaphore.acquire() because the retry fires synchronously on the IO thread before onComplete can release().

Stack Trace:
java.lang.AssertionError: BUG (LBJettySolrClient retry deadlock): futures did not complete within 30s. IO threads are permanently blocked in semaphore.acquire() because the retry fires synchronously on the IO thread before onComplete can release().
	at __randomizedtesting.SeedInfo.seed([EEC8FBE27CD386FB:3D68000916BBDE79]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.solr.handler.component.AsyncTrackerSemaphoreLeakTest.testSemaphoreLeakOnLBRetry(AsyncTrackerSemaphoreLeakTest.java:201)
	at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103)
	at java.base/java.lang.reflect.Method.invoke(Method.java:580)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1763)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
	at java.base/java.lang.Thread.run(Thread.java:1570)

[JENKINS] Solr-main-Linux (64bit/hotspot/jdk-25.0.2) - Build # 33380 - Still Unstable!

Posted by Policeman Jenkins Server via builds <bu...@solr.apache.org>.
Build: https://jenkins.thetaphi.de/job/Solr-main-Linux/33380/
Java: 64bit/hotspot/jdk-25.0.2 -XX:+UseCompressedOops -XX:+UseZGC

1 tests failed.
FAILED:  org.apache.solr.crossdc.manager.consumer.KafkaCrossDcConsumerTest.testSolrClientSupplier

Error Message:
java.lang.OutOfMemoryError: Java heap space

Stack Trace:
java.lang.OutOfMemoryError: Java heap space

[JENKINS-EA] Solr-main-Linux (64bit/hotspot/jdk-27-ea+14) - Build # 33379 - Still Unstable!

Posted by Policeman Jenkins Server via builds <bu...@solr.apache.org>.
Build: https://jenkins.thetaphi.de/job/Solr-main-Linux/33379/
Java: 64bit/hotspot/jdk-27-ea+14 -XX:-UseCompressedOops -XX:+UseSerialGC

4 tests failed.
FAILED:  org.apache.solr.cloud.BasicDistributedZkTest.test

Error Message:
java.lang.RuntimeException: Timeout waiting for pool to shutdown. [poolName=closeThreadPool][poolName=closeThreadPool] org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor@25b5789e[Shutting down, pool size = 1, active threads = 1, queued tasks = 0, completed tasks = 9]

Stack Trace:
java.lang.RuntimeException: Timeout waiting for pool to shutdown. [poolName=closeThreadPool][poolName=closeThreadPool] org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor@25b5789e[Shutting down, pool size = 1, active threads = 1, queued tasks = 0, completed tasks = 9]
	at __randomizedtesting.SeedInfo.seed([9323E3A1F71CCCE:816601E0B18DA136]:0)
	at org.apache.solr.common.util.ExecutorUtil.awaitTermination(ExecutorUtil.java:167)
	at org.apache.solr.common.util.ExecutorUtil.awaitTermination(ExecutorUtil.java:153)
	at org.apache.solr.common.util.ExecutorUtil.shutdownAndAwaitTermination(ExecutorUtil.java:119)
	at org.apache.solr.BaseDistributedSearchTestCase.destroyServers(BaseDistributedSearchTestCase.java:371)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.destroyServers(AbstractFullDistribZkTestBase.java:2269)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1065)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1034)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
	at java.base/java.lang.Thread.run(Thread.java:1527)


FAILED:  org.apache.solr.cloud.api.collections.TestReplicaProperties.classMethod

Error Message:
java.lang.AssertionError: ObjectTracker found 2 object(s) that were not released!!! [HttpJettySolrClient, SolrMetricsContext]
org.apache.solr.client.solrj.jetty.HttpJettySolrClient:org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.client.solrj.jetty.HttpJettySolrClient@932472035
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:57)
	at org.apache.solr.client.solrj.jetty.HttpJettySolrClient.<init>(HttpJettySolrClient.java:182)
	at org.apache.solr.client.solrj.jetty.HttpJettySolrClient$Builder.build(HttpJettySolrClient.java:1050)
	at org.apache.solr.handler.component.HttpShardHandlerFactory.init(HttpShardHandlerFactory.java:312)
	at org.apache.solr.handler.component.ShardHandlerFactory.newInstance(ShardHandlerFactory.java:54)
	at org.apache.solr.core.CoreContainer.loadInternal(CoreContainer.java:794)
	at org.apache.solr.core.CoreContainer.load(CoreContainer.java:753)
	at org.apache.solr.servlet.CoreContainerProvider.createCoreContainer(CoreContainerProvider.java:341)
	at org.apache.solr.servlet.CoreContainerProvider.init(CoreContainerProvider.java:181)
	at org.apache.solr.servlet.CoreContainerProvider.contextInitialized(CoreContainerProvider.java:84)
	at org.apache.solr.embedded.JettySolrRunner$1.contextInitialized(JettySolrRunner.java:319)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.callContextInitialized(ServletContextHandler.java:1592)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.contextInitialized(ServletContextHandler.java:500)
	at org.eclipse.jetty.ee10.servlet.ServletHandler.initialize(ServletHandler.java:678)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.startContext(ServletContextHandler.java:1326)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.lambda$doStart$0(ServletContextHandler.java:1052)
	at org.eclipse.jetty.server.handler.ContextHandler$ScopedContext.call(ContextHandler.java:1494)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.doStart(ServletContextHandler.java:1049)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:120)
	at org.eclipse.jetty.server.Handler$Abstract.doStart(Handler.java:491)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:120)
	at org.eclipse.jetty.server.Handler$Abstract.doStart(Handler.java:491)
	at org.eclipse.jetty.server.handler.GracefulHandler.doStart(GracefulHandler.java:134)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
	at org.eclipse.jetty.server.Server.start(Server.java:643)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:120)
	at org.eclipse.jetty.server.Handler$Abstract.doStart(Handler.java:491)
	at org.eclipse.jetty.server.Server.doStart(Server.java:584)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.apache.solr.embedded.JettySolrRunner.retryOnPortBindFailure(JettySolrRunner.java:531)
	at org.apache.solr.embedded.JettySolrRunner.start(JettySolrRunner.java:474)
	at org.apache.solr.embedded.JettySolrRunner.start(JettySolrRunner.java:446)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:379)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1061)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1034)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
	at java.base/java.lang.Thread.run(Thread.java:1527)

org.apache.solr.metrics.SolrMetricsContext:org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.handler.component.HttpShardHandlerFactory@3901f60d@956429837
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:65)
	at org.apache.solr.metrics.SolrMetricsContext.<init>(SolrMetricsContext.java:67)
	at org.apache.solr.metrics.SolrMetricsContext.getChildContext(SolrMetricsContext.java:88)
	at org.apache.solr.handler.component.HttpShardHandlerFactory.initializeMetrics(HttpShardHandlerFactory.java:457)
	at org.apache.solr.core.CoreContainer.loadInternal(CoreContainer.java:796)
	at org.apache.solr.core.CoreContainer.load(CoreContainer.java:753)
	at org.apache.solr.servlet.CoreContainerProvider.createCoreContainer(CoreContainerProvider.java:341)
	at org.apache.solr.servlet.CoreContainerProvider.init(CoreContainerProvider.java:181)
	at org.apache.solr.servlet.CoreContainerProvider.contextInitialized(CoreContainerProvider.java:84)
	at org.apache.solr.embedded.JettySolrRunner$1.contextInitialized(JettySolrRunner.java:319)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.callContextInitialized(ServletContextHandler.java:1592)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.contextInitialized(ServletContextHandler.java:500)
	at org.eclipse.jetty.ee10.servlet.ServletHandler.initialize(ServletHandler.java:678)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.startContext(ServletContextHandler.java:1326)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.lambda$doStart$0(ServletContextHandler.java:1052)
	at org.eclipse.jetty.server.handler.ContextHandler$ScopedContext.call(ContextHandler.java:1494)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.doStart(ServletContextHandler.java:1049)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:120)
	at org.eclipse.jetty.server.Handler$Abstract.doStart(Handler.java:491)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:120)
	at org.eclipse.jetty.server.Handler$Abstract.doStart(Handler.java:491)
	at org.eclipse.jetty.server.handler.GracefulHandler.doStart(GracefulHandler.java:134)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
	at org.eclipse.jetty.server.Server.start(Server.java:643)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:120)
	at org.eclipse.jetty.server.Handler$Abstract.doStart(Handler.java:491)
	at org.eclipse.jetty.server.Server.doStart(Server.java:584)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.apache.solr.embedded.JettySolrRunner.retryOnPortBindFailure(JettySolrRunner.java:531)
	at org.apache.solr.embedded.JettySolrRunner.start(JettySolrRunner.java:474)
	at org.apache.solr.embedded.JettySolrRunner.start(JettySolrRunner.java:446)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:379)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1061)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1034)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
	at java.base/java.lang.Thread.run(Thread.java:1527)

 expected null, but was:<ObjectTracker found 2 object(s) that were not released!!! [HttpJettySolrClient, SolrMetricsContext]
org.apache.solr.client.solrj.jetty.HttpJettySolrClient:org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.client.solrj.jetty.HttpJettySolrClient@932472035
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:57)
	at org.apache.solr.client.solrj.jetty.HttpJettySolrClient.<init>(HttpJettySolrClient.java:182)
	at org.apache.solr.client.solrj.jetty.HttpJettySolrClient$Builder.build(HttpJettySolrClient.java:1050)
	at org.apache.solr.handler.component.HttpShardHandlerFactory.init(HttpShardHandlerFactory.java:312)
	at org.apache.solr.handler.component.ShardHandlerFactory.newInstance(ShardHandlerFactory.java:54)
	at org.apache.solr.core.CoreContainer.loadInternal(CoreContainer.java:794)
	at org.apache.solr.core.CoreContainer.load(CoreContainer.java:753)
	at org.apache.solr.servlet.CoreContainerProvider.createCoreContainer(CoreContainerProvider.java:341)
	at org.apache.solr.servlet.CoreContainerProvider.init(CoreContainerProvider.java:181)
	at org.apache.solr.servlet.CoreContainerProvider.contextInitialized(CoreContainerProvider.java:84)
	at org.apache.solr.embedded.JettySolrRunner$1.contextInitialized(JettySolrRunner.java:319)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.callContextInitialized(ServletContextHandler.java:1592)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.contextInitialized(ServletContextHandler.java:500)
	at org.eclipse.jetty.ee10.servlet.ServletHandler.initialize(ServletHandler.java:678)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.startContext(ServletContextHandler.java:1326)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.lambda$doStart$0(ServletContextHandler.java:1052)
	at org.eclipse.jetty.server.handler.ContextHandler$ScopedContext.call(ContextHandler.java:1494)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.doStart(ServletContextHandler.java:1049)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:120)
	at org.eclipse.jetty.server.Handler$Abstract.doStart(Handler.java:491)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:120)
	at org.eclipse.jetty.server.Handler$Abstract.doStart(Handler.java:491)
	at org.eclipse.jetty.server.handler.GracefulHandler.doStart(GracefulHandler.java:134)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
	at org.eclipse.jetty.server.Server.start(Server.java:643)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:120)
	at org.eclipse.jetty.server.Handler$Abstract.doStart(Handler.java:491)
	at org.eclipse.jetty.server.Server.doStart(Server.java:584)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.apache.solr.embedded.JettySolrRunner.retryOnPortBindFailure(JettySolrRunner.java:531)
	at org.apache.solr.embedded.JettySolrRunner.start(JettySolrRunner.java:474)
	at org.apache.solr.embedded.JettySolrRunner.start(JettySolrRunner.java:446)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:379)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1061)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1034)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
	at java.base/java.lang.Thread.run(Thread.java:1527)

org.apache.solr.metrics.SolrMetricsContext:org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.handler.component.HttpShardHandlerFactory@3901f60d@956429837
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:65)
	at org.apache.solr.metrics.SolrMetricsContext.<init>(SolrMetricsContext.java:67)
	at org.apache.solr.metrics.SolrMetricsContext.getChildContext(SolrMetricsContext.java:88)
	at org.apache.solr.handler.component.HttpShardHandlerFactory.initializeMetrics(HttpShardHandlerFactory.java:457)
	at org.apache.solr.core.CoreContainer.loadInternal(CoreContainer.java:796)
	at org.apache.solr.core.CoreContainer.load(CoreContainer.java:753)
	at org.apache.solr.servlet.CoreContainerProvider.createCoreContainer(CoreContainerProvider.java:341)
	at org.apache.solr.servlet.CoreContainerProvider.init(CoreContainerProvider.java:181)
	at org.apache.solr.servlet.CoreContainerProvider.contextInitialized(CoreContainerProvider.java:84)
	at org.apache.solr.embedded.JettySolrRunner$1.contextInitialized(JettySolrRunner.java:319)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.callContextInitialized(ServletContextHandler.java:1592)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.contextInitialized(ServletContextHandler.java:500)
	at org.eclipse.jetty.ee10.servlet.ServletHandler.initialize(ServletHandler.java:678)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.startContext(ServletContextHandler.java:1326)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.lambda$doStart$0(ServletContextHandler.java:1052)
	at org.eclipse.jetty.server.handler.ContextHandler$ScopedContext.call(ContextHandler.java:1494)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.doStart(ServletContextHandler.java:1049)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:120)
	at org.eclipse.jetty.server.Handler$Abstract.doStart(Handler.java:491)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:120)
	at org.eclipse.jetty.server.Handler$Abstract.doStart(Handler.java:491)
	at org.eclipse.jetty.server.handler.GracefulHandler.doStart(GracefulHandler.java:134)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
	at org.eclipse.jetty.server.Server.start(Server.java:643)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:120)
	at org.eclipse.jetty.server.Handler$Abstract.doStart(Handler.java:491)
	at org.eclipse.jetty.server.Server.doStart(Server.java:584)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.apache.solr.embedded.JettySolrRunner.retryOnPortBindFailure(JettySolrRunner.java:531)
	at org.apache.solr.embedded.JettySolrRunner.start(JettySolrRunner.java:474)
	at org.apache.solr.embedded.JettySolrRunner.start(JettySolrRunner.java:446)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:379)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1061)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1034)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
	at java.base/java.lang.Thread.run(Thread.java:1527)

>

Stack Trace:
java.lang.AssertionError: ObjectTracker found 2 object(s) that were not released!!! [HttpJettySolrClient, SolrMetricsContext]
org.apache.solr.client.solrj.jetty.HttpJettySolrClient:org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.client.solrj.jetty.HttpJettySolrClient@932472035
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:57)
	at org.apache.solr.client.solrj.jetty.HttpJettySolrClient.<init>(HttpJettySolrClient.java:182)
	at org.apache.solr.client.solrj.jetty.HttpJettySolrClient$Builder.build(HttpJettySolrClient.java:1050)
	at org.apache.solr.handler.component.HttpShardHandlerFactory.init(HttpShardHandlerFactory.java:312)
	at org.apache.solr.handler.component.ShardHandlerFactory.newInstance(ShardHandlerFactory.java:54)
	at org.apache.solr.core.CoreContainer.loadInternal(CoreContainer.java:794)
	at org.apache.solr.core.CoreContainer.load(CoreContainer.java:753)
	at org.apache.solr.servlet.CoreContainerProvider.createCoreContainer(CoreContainerProvider.java:341)
	at org.apache.solr.servlet.CoreContainerProvider.init(CoreContainerProvider.java:181)
	at org.apache.solr.servlet.CoreContainerProvider.contextInitialized(CoreContainerProvider.java:84)
	at org.apache.solr.embedded.JettySolrRunner$1.contextInitialized(JettySolrRunner.java:319)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.callContextInitialized(ServletContextHandler.java:1592)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.contextInitialized(ServletContextHandler.java:500)
	at org.eclipse.jetty.ee10.servlet.ServletHandler.initialize(ServletHandler.java:678)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.startContext(ServletContextHandler.java:1326)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.lambda$doStart$0(ServletContextHandler.java:1052)
	at org.eclipse.jetty.server.handler.ContextHandler$ScopedContext.call(ContextHandler.java:1494)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.doStart(ServletContextHandler.java:1049)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:120)
	at org.eclipse.jetty.server.Handler$Abstract.doStart(Handler.java:491)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:120)
	at org.eclipse.jetty.server.Handler$Abstract.doStart(Handler.java:491)
	at org.eclipse.jetty.server.handler.GracefulHandler.doStart(GracefulHandler.java:134)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
	at org.eclipse.jetty.server.Server.start(Server.java:643)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:120)
	at org.eclipse.jetty.server.Handler$Abstract.doStart(Handler.java:491)
	at org.eclipse.jetty.server.Server.doStart(Server.java:584)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.apache.solr.embedded.JettySolrRunner.retryOnPortBindFailure(JettySolrRunner.java:531)
	at org.apache.solr.embedded.JettySolrRunner.start(JettySolrRunner.java:474)
	at org.apache.solr.embedded.JettySolrRunner.start(JettySolrRunner.java:446)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:379)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1061)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1034)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
	at java.base/java.lang.Thread.run(Thread.java:1527)

org.apache.solr.metrics.SolrMetricsContext:org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.handler.component.HttpShardHandlerFactory@3901f60d@956429837
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:65)
	at org.apache.solr.metrics.SolrMetricsContext.<init>(SolrMetricsContext.java:67)
	at org.apache.solr.metrics.SolrMetricsContext.getChildContext(SolrMetricsContext.java:88)
	at org.apache.solr.handler.component.HttpShardHandlerFactory.initializeMetrics(HttpShardHandlerFactory.java:457)
	at org.apache.solr.core.CoreContainer.loadInternal(CoreContainer.java:796)
	at org.apache.solr.core.CoreContainer.load(CoreContainer.java:753)
	at org.apache.solr.servlet.CoreContainerProvider.createCoreContainer(CoreContainerProvider.java:341)
	at org.apache.solr.servlet.CoreContainerProvider.init(CoreContainerProvider.java:181)
	at org.apache.solr.servlet.CoreContainerProvider.contextInitialized(CoreContainerProvider.java:84)
	at org.apache.solr.embedded.JettySolrRunner$1.contextInitialized(JettySolrRunner.java:319)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.callContextInitialized(ServletContextHandler.java:1592)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.contextInitialized(ServletContextHandler.java:500)
	at org.eclipse.jetty.ee10.servlet.ServletHandler.initialize(ServletHandler.java:678)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.startContext(ServletContextHandler.java:1326)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.lambda$doStart$0(ServletContextHandler.java:1052)
	at org.eclipse.jetty.server.handler.ContextHandler$ScopedContext.call(ContextHandler.java:1494)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.doStart(ServletContextHandler.java:1049)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:120)
	at org.eclipse.jetty.server.Handler$Abstract.doStart(Handler.java:491)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:120)
	at org.eclipse.jetty.server.Handler$Abstract.doStart(Handler.java:491)
	at org.eclipse.jetty.server.handler.GracefulHandler.doStart(GracefulHandler.java:134)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
	at org.eclipse.jetty.server.Server.start(Server.java:643)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:120)
	at org.eclipse.jetty.server.Handler$Abstract.doStart(Handler.java:491)
	at org.eclipse.jetty.server.Server.doStart(Server.java:584)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.apache.solr.embedded.JettySolrRunner.retryOnPortBindFailure(JettySolrRunner.java:531)
	at org.apache.solr.embedded.JettySolrRunner.start(JettySolrRunner.java:474)
	at org.apache.solr.embedded.JettySolrRunner.start(JettySolrRunner.java:446)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:379)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1061)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1034)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
	at java.base/java.lang.Thread.run(Thread.java:1527)

 expected null, but was:<ObjectTracker found 2 object(s) that were not released!!! [HttpJettySolrClient, SolrMetricsContext]
org.apache.solr.client.solrj.jetty.HttpJettySolrClient:org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.client.solrj.jetty.HttpJettySolrClient@932472035
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:57)
	at org.apache.solr.client.solrj.jetty.HttpJettySolrClient.<init>(HttpJettySolrClient.java:182)
	at org.apache.solr.client.solrj.jetty.HttpJettySolrClient$Builder.build(HttpJettySolrClient.java:1050)
	at org.apache.solr.handler.component.HttpShardHandlerFactory.init(HttpShardHandlerFactory.java:312)
	at org.apache.solr.handler.component.ShardHandlerFactory.newInstance(ShardHandlerFactory.java:54)
	at org.apache.solr.core.CoreContainer.loadInternal(CoreContainer.java:794)
	at org.apache.solr.core.CoreContainer.load(CoreContainer.java:753)
	at org.apache.solr.servlet.CoreContainerProvider.createCoreContainer(CoreContainerProvider.java:341)
	at org.apache.solr.servlet.CoreContainerProvider.init(CoreContainerProvider.java:181)
	at org.apache.solr.servlet.CoreContainerProvider.contextInitialized(CoreContainerProvider.java:84)
	at org.apache.solr.embedded.JettySolrRunner$1.contextInitialized(JettySolrRunner.java:319)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.callContextInitialized(ServletContextHandler.java:1592)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.contextInitialized(ServletContextHandler.java:500)
	at org.eclipse.jetty.ee10.servlet.ServletHandler.initialize(ServletHandler.java:678)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.startContext(ServletContextHandler.java:1326)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.lambda$doStart$0(ServletContextHandler.java:1052)
	at org.eclipse.jetty.server.handler.ContextHandler$ScopedContext.call(ContextHandler.java:1494)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.doStart(ServletContextHandler.java:1049)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:120)
	at org.eclipse.jetty.server.Handler$Abstract.doStart(Handler.java:491)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:120)
	at org.eclipse.jetty.server.Handler$Abstract.doStart(Handler.java:491)
	at org.eclipse.jetty.server.handler.GracefulHandler.doStart(GracefulHandler.java:134)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
	at org.eclipse.jetty.server.Server.start(Server.java:643)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:120)
	at org.eclipse.jetty.server.Handler$Abstract.doStart(Handler.java:491)
	at org.eclipse.jetty.server.Server.doStart(Server.java:584)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.apache.solr.embedded.JettySolrRunner.retryOnPortBindFailure(JettySolrRunner.java:531)
	at org.apache.solr.embedded.JettySolrRunner.start(JettySolrRunner.java:474)
	at org.apache.solr.embedded.JettySolrRunner.start(JettySolrRunner.java:446)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:379)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1061)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1034)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
	at java.base/java.lang.Thread.run(Thread.java:1527)

org.apache.solr.metrics.SolrMetricsContext:org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.handler.component.HttpShardHandlerFactory@3901f60d@956429837
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:65)
	at org.apache.solr.metrics.SolrMetricsContext.<init>(SolrMetricsContext.java:67)
	at org.apache.solr.metrics.SolrMetricsContext.getChildContext(SolrMetricsContext.java:88)
	at org.apache.solr.handler.component.HttpShardHandlerFactory.initializeMetrics(HttpShardHandlerFactory.java:457)
	at org.apache.solr.core.CoreContainer.loadInternal(CoreContainer.java:796)
	at org.apache.solr.core.CoreContainer.load(CoreContainer.java:753)
	at org.apache.solr.servlet.CoreContainerProvider.createCoreContainer(CoreContainerProvider.java:341)
	at org.apache.solr.servlet.CoreContainerProvider.init(CoreContainerProvider.java:181)
	at org.apache.solr.servlet.CoreContainerProvider.contextInitialized(CoreContainerProvider.java:84)
	at org.apache.solr.embedded.JettySolrRunner$1.contextInitialized(JettySolrRunner.java:319)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.callContextInitialized(ServletContextHandler.java:1592)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.contextInitialized(ServletContextHandler.java:500)
	at org.eclipse.jetty.ee10.servlet.ServletHandler.initialize(ServletHandler.java:678)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.startContext(ServletContextHandler.java:1326)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.lambda$doStart$0(ServletContextHandler.java:1052)
	at org.eclipse.jetty.server.handler.ContextHandler$ScopedContext.call(ContextHandler.java:1494)
	at org.eclipse.jetty.ee10.servlet.ServletContextHandler.doStart(ServletContextHandler.java:1049)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:120)
	at org.eclipse.jetty.server.Handler$Abstract.doStart(Handler.java:491)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:120)
	at org.eclipse.jetty.server.Handler$Abstract.doStart(Handler.java:491)
	at org.eclipse.jetty.server.handler.GracefulHandler.doStart(GracefulHandler.java:134)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
	at org.eclipse.jetty.server.Server.start(Server.java:643)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:120)
	at org.eclipse.jetty.server.Handler$Abstract.doStart(Handler.java:491)
	at org.eclipse.jetty.server.Server.doStart(Server.java:584)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:93)
	at org.apache.solr.embedded.JettySolrRunner.retryOnPortBindFailure(JettySolrRunner.java:531)
	at org.apache.solr.embedded.JettySolrRunner.start(JettySolrRunner.java:474)
	at org.apache.solr.embedded.JettySolrRunner.start(JettySolrRunner.java:446)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:379)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1061)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1034)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
	at java.base/java.lang.Thread.run(Thread.java:1527)

>
	at org.junit.Assert.fail(Assert.java:89)
	at org.junit.Assert.failNotNull(Assert.java:756)
	at org.junit.Assert.assertNull(Assert.java:738)
	at org.apache.solr.SolrTestCase$1.afterIfSuccessful(SolrTestCase.java:100)
	at com.carrotsearch.randomizedtesting.rules.TestRuleAdapter$1.afterIfSuccessful(TestRuleAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:37)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
	at java.base/java.lang.Thread.run(Thread.java:1527)


FAILED:  org.apache.solr.cloud.api.collections.TestReplicaProperties.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.cloud.api.collections.TestReplicaProperties: 
   1) Thread[id=10931, name=httpShardExecutor-5822-thread-2, state=RUNNABLE, group=TGRP-TestReplicaProperties]
        at java.base/sun.nio.ch.EPoll.wait(Native Method)
        at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:117)
        at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
        at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
        at org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:182)
        at org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:189)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:616)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:550)
        at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produceTask(AdaptiveExecutionStrategy.java:526)
        at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:251)
        at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produce(AdaptiveExecutionStrategy.java:196)
        at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:408)
        at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1090)
        at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
        at java.base/java.lang.Thread.run(Thread.java:1527)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.cloud.api.collections.TestReplicaProperties: 
   1) Thread[id=10931, name=httpShardExecutor-5822-thread-2, state=RUNNABLE, group=TGRP-TestReplicaProperties]
        at java.base/sun.nio.ch.EPoll.wait(Native Method)
        at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:117)
        at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
        at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
        at org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:182)
        at org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:189)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:616)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:550)
        at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produceTask(AdaptiveExecutionStrategy.java:526)
        at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:251)
        at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produce(AdaptiveExecutionStrategy.java:196)
        at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:408)
        at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1090)
        at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
        at java.base/java.lang.Thread.run(Thread.java:1527)
	at __randomizedtesting.SeedInfo.seed([9323E3A1F71CCCE]:0)


FAILED:  org.apache.solr.crossdc.manager.SolrAndKafkaIntegrationTest.testPartitioning

Error Message:
java.util.ConcurrentModificationException

Stack Trace:
java.util.ConcurrentModificationException
	at __randomizedtesting.SeedInfo.seed([9323E3A1F71CCCE:55232CF01B9B0619]:0)
	at java.base/java.util.ArrayList$Itr.checkForComodification(ArrayList.java:1104)
	at java.base/java.util.ArrayList$Itr.next(ArrayList.java:1058)
	at org.apache.solr.crossdc.manager.SolrAndKafkaIntegrationTest.testPartitioning(SolrAndKafkaIntegrationTest.java:363)
	at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
	at java.base/java.lang.reflect.Method.invoke(Method.java:565)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1763)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
	at java.base/java.lang.Thread.run(Thread.java:1527)

[JENKINS-EA] Solr-main-Linux (64bit/hotspot/jdk-27-ea+14) - Build # 33378 - Still Unstable!

Posted by Policeman Jenkins Server via builds <bu...@solr.apache.org>.
Build: https://jenkins.thetaphi.de/job/Solr-main-Linux/33378/
Java: 64bit/hotspot/jdk-27-ea+14 -XX:-UseCompressedOops -XX:+UseZGC

3 tests failed.
FAILED:  org.apache.solr.cloud.BasicDistributedZk2Test.classMethod

Error Message:
java.lang.AssertionError: ObjectTracker found 3 object(s) that were not released!!! [TransactionLog, TransactionLog, TransactionLog]
org.apache.solr.update.TransactionLog:org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.update.TransactionLog@1666805101
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:57)
	at org.apache.solr.update.TransactionLog.<init>(TransactionLog.java:256)
	at org.apache.solr.update.TransactionLog.<init>(TransactionLog.java:179)
	at org.apache.solr.update.UpdateLog.newTransactionLog(UpdateLog.java:715)
	at org.apache.solr.update.UpdateLog.ensureLog(UpdateLog.java:1677)
	at org.apache.solr.update.UpdateLog.add(UpdateLog.java:820)
	at org.apache.solr.update.UpdateLog.add(UpdateLog.java:800)
	at org.apache.solr.update.DirectUpdateHandler2.doNormalUpdate(DirectUpdateHandler2.java:535)
	at org.apache.solr.update.DirectUpdateHandler2.addDoc0(DirectUpdateHandler2.java:481)
	at org.apache.solr.update.DirectUpdateHandler2.addDoc(DirectUpdateHandler2.java:410)
	at org.apache.solr.update.processor.RunUpdateProcessorFactory$RunUpdateProcessor.processAdd(RunUpdateProcessorFactory.java:76)
	at org.apache.solr.update.processor.UpdateRequestProcessor.processAdd(UpdateRequestProcessor.java:54)
	at org.apache.solr.update.processor.NestedUpdateProcessorFactory$NestedUpdateProcessor.processAdd(NestedUpdateProcessorFactory.java:87)
	at org.apache.solr.update.processor.UpdateRequestProcessor.processAdd(UpdateRequestProcessor.java:54)
	at org.apache.solr.update.processor.DistributedUpdateProcessor.doLocalAdd(DistributedUpdateProcessor.java:274)
	at org.apache.solr.update.processor.DistributedUpdateProcessor.doVersionAdd(DistributedUpdateProcessor.java:520)
	at org.apache.solr.update.processor.DistributedUpdateProcessor.lambda$versionAdd$0(DistributedUpdateProcessor.java:361)
	at org.apache.solr.update.UpdateLocks.runWithLockInternal(UpdateLocks.java:158)
	at org.apache.solr.update.UpdateLocks.runWithLock(UpdateLocks.java:100)
	at org.apache.solr.update.processor.DistributedUpdateProcessor.versionAdd(DistributedUpdateProcessor.java:346)
	at org.apache.solr.update.processor.DistributedUpdateProcessor.processAdd(DistributedUpdateProcessor.java:240)
	at org.apache.solr.update.processor.DistributedZkUpdateProcessor.processAdd(DistributedZkUpdateProcessor.java:280)
	at org.apache.solr.update.processor.LogUpdateProcessorFactory$LogUpdateProcessor.processAdd(LogUpdateProcessorFactory.java:111)
	at org.apache.solr.handler.loader.JavabinLoader$1.update(JavabinLoader.java:122)
	at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$StreamingCodec.readDocs(JavaBinUpdateRequestCodec.java:259)
	at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$StreamingCodec.readIterator(JavaBinUpdateRequestCodec.java:214)
	at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:345)
	at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:290)
	at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$StreamingCodec.readNamedList(JavaBinUpdateRequestCodec.java:200)
	at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:310)
	at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:290)
	at org.apache.solr.common.util.JavaBinCodec.unmarshal(JavaBinCodec.java:199)
	at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$StreamingCodec.unmarshal(JavaBinUpdateRequestCodec.java:182)
	at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.unmarshal(JavaBinUpdateRequestCodec.java:110)
	at org.apache.solr.handler.loader.JavabinLoader.parseAndLoadDocs(JavabinLoader.java:134)
	at org.apache.solr.handler.loader.JavabinLoader.load(JavabinLoader.java:73)
	at org.apache.solr.handler.UpdateRequestHandler$1.load(UpdateRequestHandler.java:101)
	at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:99)
	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:251)
	at org.apache.solr.core.SolrCore.execute(SolrCore.java:2919)
	at org.apache.solr.servlet.HttpSolrCall.executeCoreRequest(HttpSolrCall.java:699)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:492)
	at org.apache.solr.servlet.SolrServlet.dispatch(SolrServlet.java:114)
	at org.apache.solr.servlet.SolrServlet.service(SolrServlet.java:90)
	at jakarta.servlet.http.HttpServlet.service(HttpServlet.java:723)
	at org.eclipse.jetty.ee10.servlet.ServletHolder.handle(ServletHolder.java:753)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1624)
	at org.apache.solr.servlet.AuthenticationFilter.doFilter(AuthenticationFilter.java:79)
	at jakarta.servlet.http.HttpFilter.doFilter(HttpFilter.java:97)
	at org.eclipse.jetty.ee10.servlet.FilterHolder.doFilter(FilterHolder.java:205)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1596)
	at org.apache.solr.servlet.TracingFilter.doFilter(TracingFilter.java:55)
	at jakarta.servlet.http.HttpFilter.doFilter(HttpFilter.java:97)
	at org.eclipse.jetty.ee10.servlet.FilterHolder.doFilter(FilterHolder.java:205)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1596)
	at org.apache.solr.servlet.RateLimitFilter.doFilter(RateLimitFilter.java:50)
	at jakarta.servlet.http.HttpFilter.doFilter(HttpFilter.java:97)
	at org.eclipse.jetty.ee10.servlet.FilterHolder.doFilter(FilterHolder.java:205)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1596)
	at org.apache.solr.servlet.RequiredSolrRequestFilter.doFilter(RequiredSolrRequestFilter.java:79)
	at jakarta.servlet.http.HttpFilter.doFilter(HttpFilter.java:97)
	at org.eclipse.jetty.ee10.servlet.FilterHolder.doFilter(FilterHolder.java:205)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1596)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$MappedServlet.handle(ServletHandler.java:1557)
	at org.eclipse.jetty.ee10.servlet.ServletChannel.dispatch(ServletChannel.java:828)
	at org.eclipse.jetty.ee10.servlet.ServletChannel.handle(ServletChannel.java:438)
	at org.eclipse.jetty.ee10.servlet.ServletHandler.handle(ServletHandler.java:470)
	at org.eclipse.jetty.server.handler.ContextHandler.handle(ContextHandler.java:1102)
	at org.eclipse.jetty.server.Handler$Wrapper.handle(Handler.java:740)
	at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:138)
	at org.eclipse.jetty.server.Handler$Wrapper.handle(Handler.java:740)
	at org.eclipse.jetty.server.handler.GracefulHandler.handle(GracefulHandler.java:112)
	at org.eclipse.jetty.server.Server.handle(Server.java:182)
	at org.eclipse.jetty.server.internal.HttpChannelState$HandlerInvoker.run(HttpChannelState.java:721)
	at org.eclipse.jetty.util.thread.Invocable$ReadyTask.run(Invocable.java:177)
	at org.eclipse.jetty.http2.server.internal.HttpStreamOverHTTP2$1.run(HttpStreamOverHTTP2.java:144)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:981)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.doRunJob(QueuedThreadPool.java:1212)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1166)
	at java.base/java.lang.Thread.run(Thread.java:1527)

org.apache.solr.update.TransactionLog:org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.update.TransactionLog@431332166
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:57)
	at org.apache.solr.update.TransactionLog.<init>(TransactionLog.java:256)
	at org.apache.solr.update.TransactionLog.<init>(TransactionLog.java:179)
	at org.apache.solr.update.UpdateLog.newTransactionLog(UpdateLog.java:715)
	at org.apache.solr.update.UpdateLog.ensureLog(UpdateLog.java:1677)
	at org.apache.solr.update.UpdateLog.add(UpdateLog.java:820)
	at org.apache.solr.update.UpdateLog.add(UpdateLog.java:800)
	at org.apache.solr.update.DirectUpdateHandler2.doNormalUpdate(DirectUpdateHandler2.java:535)
	at org.apache.solr.update.DirectUpdateHandler2.addDoc0(DirectUpdateHandler2.java:481)
	at org.apache.solr.update.DirectUpdateHandler2.addDoc(DirectUpdateHandler2.java:410)
	at org.apache.solr.update.processor.RunUpdateProcessorFactory$RunUpdateProcessor.processAdd(RunUpdateProcessorFactory.java:76)
	at org.apache.solr.update.processor.UpdateRequestProcessor.processAdd(UpdateRequestProcessor.java:54)
	at org.apache.solr.update.processor.NestedUpdateProcessorFactory$NestedUpdateProcessor.processAdd(NestedUpdateProcessorFactory.java:87)
	at org.apache.solr.update.processor.UpdateRequestProcessor.processAdd(UpdateRequestProcessor.java:54)
	at org.apache.solr.update.processor.DistributedUpdateProcessor.doLocalAdd(DistributedUpdateProcessor.java:274)
	at org.apache.solr.update.processor.DistributedUpdateProcessor.doVersionAdd(DistributedUpdateProcessor.java:520)
	at org.apache.solr.update.processor.DistributedUpdateProcessor.lambda$versionAdd$0(DistributedUpdateProcessor.java:361)
	at org.apache.solr.update.UpdateLocks.runWithLockInternal(UpdateLocks.java:158)
	at org.apache.solr.update.UpdateLocks.runWithLock(UpdateLocks.java:100)
	at org.apache.solr.update.processor.DistributedUpdateProcessor.versionAdd(DistributedUpdateProcessor.java:346)
	at org.apache.solr.update.processor.DistributedUpdateProcessor.processAdd(DistributedUpdateProcessor.java:240)
	at org.apache.solr.update.processor.DistributedZkUpdateProcessor.processAdd(DistributedZkUpdateProcessor.java:280)
	at org.apache.solr.update.processor.LogUpdateProcessorFactory$LogUpdateProcessor.processAdd(LogUpdateProcessorFactory.java:111)
	at org.apache.solr.handler.loader.JavabinLoader$1.update(JavabinLoader.java:122)
	at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$StreamingCodec.readDocs(JavaBinUpdateRequestCodec.java:259)
	at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$StreamingCodec.readIterator(JavaBinUpdateRequestCodec.java:214)
	at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:345)
	at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:290)
	at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$StreamingCodec.readNamedList(JavaBinUpdateRequestCodec.java:200)
	at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:310)
	at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:290)
	at org.apache.solr.common.util.JavaBinCodec.unmarshal(JavaBinCodec.java:199)
	at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$StreamingCodec.unmarshal(JavaBinUpdateRequestCodec.java:182)
	at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.unmarshal(JavaBinUpdateRequestCodec.java:110)
	at org.apache.solr.handler.loader.JavabinLoader.parseAndLoadDocs(JavabinLoader.java:134)
	at org.apache.solr.handler.loader.JavabinLoader.load(JavabinLoader.java:73)
	at org.apache.solr.handler.UpdateRequestHandler$1.load(UpdateRequestHandler.java:101)
	at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:99)
	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:251)
	at org.apache.solr.core.SolrCore.execute(SolrCore.java:2919)
	at org.apache.solr.servlet.HttpSolrCall.executeCoreRequest(HttpSolrCall.java:699)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:492)
	at org.apache.solr.servlet.SolrServlet.dispatch(SolrServlet.java:114)
	at org.apache.solr.servlet.SolrServlet.service(SolrServlet.java:90)
	at jakarta.servlet.http.HttpServlet.service(HttpServlet.java:723)
	at org.eclipse.jetty.ee10.servlet.ServletHolder.handle(ServletHolder.java:753)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1624)
	at org.apache.solr.servlet.AuthenticationFilter.doFilter(AuthenticationFilter.java:79)
	at jakarta.servlet.http.HttpFilter.doFilter(HttpFilter.java:97)
	at org.eclipse.jetty.ee10.servlet.FilterHolder.doFilter(FilterHolder.java:205)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1596)
	at org.apache.solr.servlet.TracingFilter.doFilter(TracingFilter.java:55)
	at jakarta.servlet.http.HttpFilter.doFilter(HttpFilter.java:97)
	at org.eclipse.jetty.ee10.servlet.FilterHolder.doFilter(FilterHolder.java:205)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1596)
	at org.apache.solr.servlet.RateLimitFilter.doFilter(RateLimitFilter.java:50)
	at jakarta.servlet.http.HttpFilter.doFilter(HttpFilter.java:97)
	at org.eclipse.jetty.ee10.servlet.FilterHolder.doFilter(FilterHolder.java:205)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1596)
	at org.apache.solr.servlet.RequiredSolrRequestFilter.doFilter(RequiredSolrRequestFilter.java:79)
	at jakarta.servlet.http.HttpFilter.doFilter(HttpFilter.java:97)
	at org.eclipse.jetty.ee10.servlet.FilterHolder.doFilter(FilterHolder.java:205)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1596)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$MappedServlet.handle(ServletHandler.java:1557)
	at org.eclipse.jetty.ee10.servlet.ServletChannel.dispatch(ServletChannel.java:828)
	at org.eclipse.jetty.ee10.servlet.ServletChannel.handle(ServletChannel.java:438)
	at org.eclipse.jetty.ee10.servlet.ServletHandler.handle(ServletHandler.java:470)
	at org.eclipse.jetty.server.handler.ContextHandler.handle(ContextHandler.java:1102)
	at org.eclipse.jetty.server.Handler$Wrapper.handle(Handler.java:740)
	at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:138)
	at org.eclipse.jetty.server.Handler$Wrapper.handle(Handler.java:740)
	at org.eclipse.jetty.server.handler.GracefulHandler.handle(GracefulHandler.java:112)
	at org.eclipse.jetty.server.Server.handle(Server.java:182)
	at org.eclipse.jetty.server.internal.HttpChannelState$HandlerInvoker.run(HttpChannelState.java:721)
	at org.eclipse.jetty.util.thread.Invocable$ReadyTask.run(Invocable.java:177)
	at org.eclipse.jetty.http2.server.internal.HttpStreamOverHTTP2$1.run(HttpStreamOverHTTP2.java:144)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:981)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.doRunJob(QueuedThreadPool.java:1212)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1166)
	at java.base/java.lang.Thread.run(Thread.java:1527)

org.apache.solr.update.TransactionLog:org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.update.TransactionLog@1098408656
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:57)
	at org.apache.solr.update.TransactionLog.<init>(TransactionLog.java:256)
	at org.apache.solr.update.TransactionLog.<init>(TransactionLog.java:179)
	at org.apache.solr.update.UpdateLog.newTransactionLog(UpdateLog.java:715)
	at org.apache.solr.update.UpdateLog.ensureLog(UpdateLog.java:1677)
	at org.apache.solr.update.UpdateLog.delete(UpdateLog.java:886)
	at org.apache.solr.update.DirectUpdateHandler2.delete(DirectUpdateHandler2.java:623)
	at org.apache.solr.update.processor.RunUpdateProcessorFactory$RunUpdateProcessor.processDelete(RunUpdateProcessorFactory.java:84)
	at org.apache.solr.update.processor.UpdateRequestProcessor.processDelete(UpdateRequestProcessor.java:58)
	at org.apache.solr.update.processor.UpdateRequestProcessor.processDelete(UpdateRequestProcessor.java:58)
	at org.apache.solr.update.processor.DistributedUpdateProcessor.doLocalDelete(DistributedUpdateProcessor.java:280)
	at org.apache.solr.update.processor.DistributedUpdateProcessor.doVersionDelete(DistributedUpdateProcessor.java:1140)
	at org.apache.solr.update.processor.DistributedUpdateProcessor.lambda$versionDelete$0(DistributedUpdateProcessor.java:1044)
	at org.apache.solr.update.UpdateLocks.runWithLockInternal(UpdateLocks.java:158)
	at org.apache.solr.update.UpdateLocks.runWithLock(UpdateLocks.java:100)
	at org.apache.solr.update.processor.DistributedUpdateProcessor.versionDelete(DistributedUpdateProcessor.java:1041)
	at org.apache.solr.update.processor.DistributedUpdateProcessor.doDeleteById(DistributedUpdateProcessor.java:811)
	at org.apache.solr.update.processor.DistributedZkUpdateProcessor.doDeleteById(DistributedZkUpdateProcessor.java:376)
	at org.apache.solr.update.processor.DistributedUpdateProcessor.processDelete(DistributedUpdateProcessor.java:797)
	at org.apache.solr.update.processor.DistributedZkUpdateProcessor.processDelete(DistributedZkUpdateProcessor.java:353)
	at org.apache.solr.update.processor.LogUpdateProcessorFactory$LogUpdateProcessor.processDelete(LogUpdateProcessorFactory.java:134)
	at org.apache.solr.handler.loader.JavabinLoader.delete(JavabinLoader.java:220)
	at org.apache.solr.handler.loader.JavabinLoader.parseAndLoadDocs(JavabinLoader.java:139)
	at org.apache.solr.handler.loader.JavabinLoader.load(JavabinLoader.java:73)
	at org.apache.solr.handler.UpdateRequestHandler$1.load(UpdateRequestHandler.java:101)
	at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:99)
	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:251)
	at org.apache.solr.core.SolrCore.execute(SolrCore.java:2919)
	at org.apache.solr.servlet.HttpSolrCall.executeCoreRequest(HttpSolrCall.java:699)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:492)
	at org.apache.solr.servlet.SolrServlet.dispatch(SolrServlet.java:114)
	at org.apache.solr.servlet.SolrServlet.service(SolrServlet.java:90)
	at jakarta.servlet.http.HttpServlet.service(HttpServlet.java:723)
	at org.eclipse.jetty.ee10.servlet.ServletHolder.handle(ServletHolder.java:753)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1624)
	at org.apache.solr.servlet.AuthenticationFilter.doFilter(AuthenticationFilter.java:79)
	at jakarta.servlet.http.HttpFilter.doFilter(HttpFilter.java:97)
	at org.eclipse.jetty.ee10.servlet.FilterHolder.doFilter(FilterHolder.java:205)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1596)
	at org.apache.solr.servlet.TracingFilter.doFilter(TracingFilter.java:55)
	at jakarta.servlet.http.HttpFilter.doFilter(HttpFilter.java:97)
	at org.eclipse.jetty.ee10.servlet.FilterHolder.doFilter(FilterHolder.java:205)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1596)
	at org.apache.solr.servlet.RateLimitFilter.doFilter(RateLimitFilter.java:50)
	at jakarta.servlet.http.HttpFilter.doFilter(HttpFilter.java:97)
	at org.eclipse.jetty.ee10.servlet.FilterHolder.doFilter(FilterHolder.java:205)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1596)
	at org.apache.solr.servlet.RequiredSolrRequestFilter.doFilter(RequiredSolrRequestFilter.java:79)
	at jakarta.servlet.http.HttpFilter.doFilter(HttpFilter.java:97)
	at org.eclipse.jetty.ee10.servlet.FilterHolder.doFilter(FilterHolder.java:205)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1596)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$MappedServlet.handle(ServletHandler.java:1557)
	at org.eclipse.jetty.ee10.servlet.ServletChannel.dispatch(ServletChannel.java:828)
	at org.eclipse.jetty.ee10.servlet.ServletChannel.handle(ServletChannel.java:438)
	at org.eclipse.jetty.ee10.servlet.ServletHandler.handle(ServletHandler.java:470)
	at org.eclipse.jetty.server.handler.ContextHandler.handle(ContextHandler.java:1102)
	at org.eclipse.jetty.server.Handler$Wrapper.handle(Handler.java:740)
	at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:138)
	at org.eclipse.jetty.server.Handler$Wrapper.handle(Handler.java:740)
	at org.eclipse.jetty.server.handler.GracefulHandler.handle(GracefulHandler.java:112)
	at org.eclipse.jetty.server.Server.handle(Server.java:182)
	at org.eclipse.jetty.server.internal.HttpChannelState$HandlerInvoker.run(HttpChannelState.java:721)
	at org.eclipse.jetty.util.thread.Invocable$ReadyTask.run(Invocable.java:177)
	at org.eclipse.jetty.http2.server.internal.HttpStreamOverHTTP2$1.run(HttpStreamOverHTTP2.java:144)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:981)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.doRunJob(QueuedThreadPool.java:1212)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1166)
	at java.base/java.lang.Thread.run(Thread.java:1527)

 expected null, but was:<ObjectTracker found 3 object(s) that were not released!!! [TransactionLog, TransactionLog, TransactionLog]
org.apache.solr.update.TransactionLog:org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.update.TransactionLog@1666805101
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:57)
	at org.apache.solr.update.TransactionLog.<init>(TransactionLog.java:256)
	at org.apache.solr.update.TransactionLog.<init>(TransactionLog.java:179)
	at org.apache.solr.update.UpdateLog.newTransactionLog(UpdateLog.java:715)
	at org.apache.solr.update.UpdateLog.ensureLog(UpdateLog.java:1677)
	at org.apache.solr.update.UpdateLog.add(UpdateLog.java:820)
	at org.apache.solr.update.UpdateLog.add(UpdateLog.java:800)
	at org.apache.solr.update.DirectUpdateHandler2.doNormalUpdate(DirectUpdateHandler2.java:535)
	at org.apache.solr.update.DirectUpdateHandler2.addDoc0(DirectUpdateHandler2.java:481)
	at org.apache.solr.update.DirectUpdateHandler2.addDoc(DirectUpdateHandler2.java:410)
	at org.apache.solr.update.processor.RunUpdateProcessorFactory$RunUpdateProcessor.processAdd(RunUpdateProcessorFactory.java:76)
	at org.apache.solr.update.processor.UpdateRequestProcessor.processAdd(UpdateRequestProcessor.java:54)
	at org.apache.solr.update.processor.NestedUpdateProcessorFactory$NestedUpdateProcessor.processAdd(NestedUpdateProcessorFactory.java:87)
	at org.apache.solr.update.processor.UpdateRequestProcessor.processAdd(UpdateRequestProcessor.java:54)
	at org.apache.solr.update.processor.DistributedUpdateProcessor.doLocalAdd(DistributedUpdateProcessor.java:274)
	at org.apache.solr.update.processor.DistributedUpdateProcessor.doVersionAdd(DistributedUpdateProcessor.java:520)
	at org.apache.solr.update.processor.DistributedUpdateProcessor.lambda$versionAdd$0(DistributedUpdateProcessor.java:361)
	at org.apache.solr.update.UpdateLocks.runWithLockInternal(UpdateLocks.java:158)
	at org.apache.solr.update.UpdateLocks.runWithLock(UpdateLocks.java:100)
	at org.apache.solr.update.processor.DistributedUpdateProcessor.versionAdd(DistributedUpdateProcessor.java:346)
	at org.apache.solr.update.processor.DistributedUpdateProcessor.processAdd(DistributedUpdateProcessor.java:240)
	at org.apache.solr.update.processor.DistributedZkUpdateProcessor.processAdd(DistributedZkUpdateProcessor.java:280)
	at org.apache.solr.update.processor.LogUpdateProcessorFactory$LogUpdateProcessor.processAdd(LogUpdateProcessorFactory.java:111)
	at org.apache.solr.handler.loader.JavabinLoader$1.update(JavabinLoader.java:122)
	at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$StreamingCodec.readDocs(JavaBinUpdateRequestCodec.java:259)
	at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$StreamingCodec.readIterator(JavaBinUpdateRequestCodec.java:214)
	at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:345)
	at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:290)
	at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$StreamingCodec.readNamedList(JavaBinUpdateRequestCodec.java:200)
	at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:310)
	at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:290)
	at org.apache.solr.common.util.JavaBinCodec.unmarshal(JavaBinCodec.java:199)
	at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$StreamingCodec.unmarshal(JavaBinUpdateRequestCodec.java:182)
	at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.unmarshal(JavaBinUpdateRequestCodec.java:110)
	at org.apache.solr.handler.loader.JavabinLoader.parseAndLoadDocs(JavabinLoader.java:134)
	at org.apache.solr.handler.loader.JavabinLoader.load(JavabinLoader.java:73)
	at org.apache.solr.handler.UpdateRequestHandler$1.load(UpdateRequestHandler.java:101)
	at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:99)
	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:251)
	at org.apache.solr.core.SolrCore.execute(SolrCore.java:2919)
	at org.apache.solr.servlet.HttpSolrCall.executeCoreRequest(HttpSolrCall.java:699)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:492)
	at org.apache.solr.servlet.SolrServlet.dispatch(SolrServlet.java:114)
	at org.apache.solr.servlet.SolrServlet.service(SolrServlet.java:90)
	at jakarta.servlet.http.HttpServlet.service(HttpServlet.java:723)
	at org.eclipse.jetty.ee10.servlet.ServletHolder.handle(ServletHolder.java:753)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1624)
	at org.apache.solr.servlet.AuthenticationFilter.doFilter(AuthenticationFilter.java:79)
	at jakarta.servlet.http.HttpFilter.doFilter(HttpFilter.java:97)
	at org.eclipse.jetty.ee10.servlet.FilterHolder.doFilter(FilterHolder.java:205)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1596)
	at org.apache.solr.servlet.TracingFilter.doFilter(TracingFilter.java:55)
	at jakarta.servlet.http.HttpFilter.doFilter(HttpFilter.java:97)
	at org.eclipse.jetty.ee10.servlet.FilterHolder.doFilter(FilterHolder.java:205)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1596)
	at org.apache.solr.servlet.RateLimitFilter.doFilter(RateLimitFilter.java:50)
	at jakarta.servlet.http.HttpFilter.doFilter(HttpFilter.java:97)
	at org.eclipse.jetty.ee10.servlet.FilterHolder.doFilter(FilterHolder.java:205)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1596)
	at org.apache.solr.servlet.RequiredSolrRequestFilter.doFilter(RequiredSolrRequestFilter.java:79)
	at jakarta.servlet.http.HttpFilter.doFilter(HttpFilter.java:97)
	at org.eclipse.jetty.ee10.servlet.FilterHolder.doFilter(FilterHolder.java:205)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1596)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$MappedServlet.handle(ServletHandler.java:1557)
	at org.eclipse.jetty.ee10.servlet.ServletChannel.dispatch(ServletChannel.java:828)
	at org.eclipse.jetty.ee10.servlet.ServletChannel.handle(ServletChannel.java:438)
	at org.eclipse.jetty.ee10.servlet.ServletHandler.handle(ServletHandler.java:470)
	at org.eclipse.jetty.server.handler.ContextHandler.handle(ContextHandler.java:1102)
	at org.eclipse.jetty.server.Handler$Wrapper.handle(Handler.java:740)
	at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:138)
	at org.eclipse.jetty.server.Handler$Wrapper.handle(Handler.java:740)
	at org.eclipse.jetty.server.handler.GracefulHandler.handle(GracefulHandler.java:112)
	at org.eclipse.jetty.server.Server.handle(Server.java:182)
	at org.eclipse.jetty.server.internal.HttpChannelState$HandlerInvoker.run(HttpChannelState.java:721)
	at org.eclipse.jetty.util.thread.Invocable$ReadyTask.run(Invocable.java:177)
	at org.eclipse.jetty.http2.server.internal.HttpStreamOverHTTP2$1.run(HttpStreamOverHTTP2.java:144)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:981)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.doRunJob(QueuedThreadPool.java:1212)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1166)
	at java.base/java.lang.Thread.run(Thread.java:1527)

org.apache.solr.update.TransactionLog:org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.update.TransactionLog@431332166
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:57)
	at org.apache.solr.update.TransactionLog.<init>(TransactionLog.java:256)
	at org.apache.solr.update.TransactionLog.<init>(TransactionLog.java:179)
	at org.apache.solr.update.UpdateLog.newTransactionLog(UpdateLog.java:715)
	at org.apache.solr.update.UpdateLog.ensureLog(UpdateLog.java:1677)
	at org.apache.solr.update.UpdateLog.add(UpdateLog.java:820)
	at org.apache.solr.update.UpdateLog.add(UpdateLog.java:800)
	at org.apache.solr.update.DirectUpdateHandler2.doNormalUpdate(DirectUpdateHandler2.java:535)
	at org.apache.solr.update.DirectUpdateHandler2.addDoc0(DirectUpdateHandler2.java:481)
	at org.apache.solr.update.DirectUpdateHandler2.addDoc(DirectUpdateHandler2.java:410)
	at org.apache.solr.update.processor.RunUpdateProcessorFactory$RunUpdateProcessor.processAdd(RunUpdateProcessorFactory.java:76)
	at org.apache.solr.update.processor.UpdateRequestProcessor.processAdd(UpdateRequestProcessor.java:54)
	at org.apache.solr.update.processor.NestedUpdateProcessorFactory$NestedUpdateProcessor.processAdd(NestedUpdateProcessorFactory.java:87)
	at org.apache.solr.update.processor.UpdateRequestProcessor.processAdd(UpdateRequestProcessor.java:54)
	at org.apache.solr.update.processor.DistributedUpdateProcessor.doLocalAdd(DistributedUpdateProcessor.java:274)
	at org.apache.solr.update.processor.DistributedUpdateProcessor.doVersionAdd(DistributedUpdateProcessor.java:520)
	at org.apache.solr.update.processor.DistributedUpdateProcessor.lambda$versionAdd$0(DistributedUpdateProcessor.java:361)
	at org.apache.solr.update.UpdateLocks.runWithLockInternal(UpdateLocks.java:158)
	at org.apache.solr.update.UpdateLocks.runWithLock(UpdateLocks.java:100)
	at org.apache.solr.update.processor.DistributedUpdateProcessor.versionAdd(DistributedUpdateProcessor.java:346)
	at org.apache.solr.update.processor.DistributedUpdateProcessor.processAdd(DistributedUpdateProcessor.java:240)
	at org.apache.solr.update.processor.DistributedZkUpdateProcessor.processAdd(DistributedZkUpdateProcessor.java:280)
	at org.apache.solr.update.processor.LogUpdateProcessorFactory$LogUpdateProcessor.processAdd(LogUpdateProcessorFactory.java:111)
	at org.apache.solr.handler.loader.JavabinLoader$1.update(JavabinLoader.java:122)
	at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$StreamingCodec.readDocs(JavaBinUpdateRequestCodec.java:259)
	at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$StreamingCodec.readIterator(JavaBinUpdateRequestCodec.java:214)
	at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:345)
	at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:290)
	at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$StreamingCodec.readNamedList(JavaBinUpdateRequestCodec.java:200)
	at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:310)
	at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:290)
	at org.apache.solr.common.util.JavaBinCodec.unmarshal(JavaBinCodec.java:199)
	at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$StreamingCodec.unmarshal(JavaBinUpdateRequestCodec.java:182)
	at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.unmarshal(JavaBinUpdateRequestCodec.java:110)
	at org.apache.solr.handler.loader.JavabinLoader.parseAndLoadDocs(JavabinLoader.java:134)
	at org.apache.solr.handler.loader.JavabinLoader.load(JavabinLoader.java:73)
	at org.apache.solr.handler.UpdateRequestHandler$1.load(UpdateRequestHandler.java:101)
	at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:99)
	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:251)
	at org.apache.solr.core.SolrCore.execute(SolrCore.java:2919)
	at org.apache.solr.servlet.HttpSolrCall.executeCoreRequest(HttpSolrCall.java:699)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:492)
	at org.apache.solr.servlet.SolrServlet.dispatch(SolrServlet.java:114)
	at org.apache.solr.servlet.SolrServlet.service(SolrServlet.java:90)
	at jakarta.servlet.http.HttpServlet.service(HttpServlet.java:723)
	at org.eclipse.jetty.ee10.servlet.ServletHolder.handle(ServletHolder.java:753)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1624)
	at org.apache.solr.servlet.AuthenticationFilter.doFilter(AuthenticationFilter.java:79)
	at jakarta.servlet.http.HttpFilter.doFilter(HttpFilter.java:97)
	at org.eclipse.jetty.ee10.servlet.FilterHolder.doFilter(FilterHolder.java:205)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1596)
	at org.apache.solr.servlet.TracingFilter.doFilter(TracingFilter.java:55)
	at jakarta.servlet.http.HttpFilter.doFilter(HttpFilter.java:97)
	at org.eclipse.jetty.ee10.servlet.FilterHolder.doFilter(FilterHolder.java:205)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1596)
	at org.apache.solr.servlet.RateLimitFilter.doFilter(RateLimitFilter.java:50)
	at jakarta.servlet.http.HttpFilter.doFilter(HttpFilter.java:97)
	at org.eclipse.jetty.ee10.servlet.FilterHolder.doFilter(FilterHolder.java:205)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1596)
	at org.apache.solr.servlet.RequiredSolrRequestFilter.doFilter(RequiredSolrRequestFilter.java:79)
	at jakarta.servlet.http.HttpFilter.doFilter(HttpFilter.java:97)
	at org.eclipse.jetty.ee10.servlet.FilterHolder.doFilter(FilterHolder.java:205)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1596)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$MappedServlet.handle(ServletHandler.java:1557)
	at org.eclipse.jetty.ee10.servlet.ServletChannel.dispatch(ServletChannel.java:828)
	at org.eclipse.jetty.ee10.servlet.ServletChannel.handle(ServletChannel.java:438)
	at org.eclipse.jetty.ee10.servlet.ServletHandler.handle(ServletHandler.java:470)
	at org.eclipse.jetty.server.handler.ContextHandler.handle(ContextHandler.java:1102)
	at org.eclipse.jetty.server.Handler$Wrapper.handle(Handler.java:740)
	at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:138)
	at org.eclipse.jetty.server.Handler$Wrapper.handle(Handler.java:740)
	at org.eclipse.jetty.server.handler.GracefulHandler.handle(GracefulHandler.java:112)
	at org.eclipse.jetty.server.Server.handle(Server.java:182)
	at org.eclipse.jetty.server.internal.HttpChannelState$HandlerInvoker.run(HttpChannelState.java:721)
	at org.eclipse.jetty.util.thread.Invocable$ReadyTask.run(Invocable.java:177)
	at org.eclipse.jetty.http2.server.internal.HttpStreamOverHTTP2$1.run(HttpStreamOverHTTP2.java:144)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:981)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.doRunJob(QueuedThreadPool.java:1212)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1166)
	at java.base/java.lang.Thread.run(Thread.java:1527)

org.apache.solr.update.TransactionLog:org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.update.TransactionLog@1098408656
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:57)
	at org.apache.solr.update.TransactionLog.<init>(TransactionLog.java:256)
	at org.apache.solr.update.TransactionLog.<init>(TransactionLog.java:179)
	at org.apache.solr.update.UpdateLog.newTransactionLog(UpdateLog.java:715)
	at org.apache.solr.update.UpdateLog.ensureLog(UpdateLog.java:1677)
	at org.apache.solr.update.UpdateLog.delete(UpdateLog.java:886)
	at org.apache.solr.update.DirectUpdateHandler2.delete(DirectUpdateHandler2.java:623)
	at org.apache.solr.update.processor.RunUpdateProcessorFactory$RunUpdateProcessor.processDelete(RunUpdateProcessorFactory.java:84)
	at org.apache.solr.update.processor.UpdateRequestProcessor.processDelete(UpdateRequestProcessor.java:58)
	at org.apache.solr.update.processor.UpdateRequestProcessor.processDelete(UpdateRequestProcessor.java:58)
	at org.apache.solr.update.processor.DistributedUpdateProcessor.doLocalDelete(DistributedUpdateProcessor.java:280)
	at org.apache.solr.update.processor.DistributedUpdateProcessor.doVersionDelete(DistributedUpdateProcessor.java:1140)
	at org.apache.solr.update.processor.DistributedUpdateProcessor.lambda$versionDelete$0(DistributedUpdateProcessor.java:1044)
	at org.apache.solr.update.UpdateLocks.runWithLockInternal(UpdateLocks.java:158)
	at org.apache.solr.update.UpdateLocks.runWithLock(UpdateLocks.java:100)
	at org.apache.solr.update.processor.DistributedUpdateProcessor.versionDelete(DistributedUpdateProcessor.java:1041)
	at org.apache.solr.update.processor.DistributedUpdateProcessor.doDeleteById(DistributedUpdateProcessor.java:811)
	at org.apache.solr.update.processor.DistributedZkUpdateProcessor.doDeleteById(DistributedZkUpdateProcessor.java:376)
	at org.apache.solr.update.processor.DistributedUpdateProcessor.processDelete(DistributedUpdateProcessor.java:797)
	at org.apache.solr.update.processor.DistributedZkUpdateProcessor.processDelete(DistributedZkUpdateProcessor.java:353)
	at org.apache.solr.update.processor.LogUpdateProcessorFactory$LogUpdateProcessor.processDelete(LogUpdateProcessorFactory.java:134)
	at org.apache.solr.handler.loader.JavabinLoader.delete(JavabinLoader.java:220)
	at org.apache.solr.handler.loader.JavabinLoader.parseAndLoadDocs(JavabinLoader.java:139)
	at org.apache.solr.handler.loader.JavabinLoader.load(JavabinLoader.java:73)
	at org.apache.solr.handler.UpdateRequestHandler$1.load(UpdateRequestHandler.java:101)
	at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:99)
	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:251)
	at org.apache.solr.core.SolrCore.execute(SolrCore.java:2919)
	at org.apache.solr.servlet.HttpSolrCall.executeCoreRequest(HttpSolrCall.java:699)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:492)
	at org.apache.solr.servlet.SolrServlet.dispatch(SolrServlet.java:114)
	at org.apache.solr.servlet.SolrServlet.service(SolrServlet.java:90)
	at jakarta.servlet.http.HttpServlet.service(HttpServlet.java:723)
	at org.eclipse.jetty.ee10.servlet.ServletHolder.handle(ServletHolder.java:753)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1624)
	at org.apache.solr.servlet.AuthenticationFilter.doFilter(AuthenticationFilter.java:79)
	at jakarta.servlet.http.HttpFilter.doFilter(HttpFilter.java:97)
	at org.eclipse.jetty.ee10.servlet.FilterHolder.doFilter(FilterHolder.java:205)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1596)
	at org.apache.solr.servlet.TracingFilter.doFilter(TracingFilter.java:55)
	at jakarta.servlet.http.HttpFilter.doFilter(HttpFilter.java:97)
	at org.eclipse.jetty.ee10.servlet.FilterHolder.doFilter(FilterHolder.java:205)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1596)
	at org.apache.solr.servlet.RateLimitFilter.doFilter(RateLimitFilter.java:50)
	at jakarta.servlet.http.HttpFilter.doFilter(HttpFilter.java:97)
	at org.eclipse.jetty.ee10.servlet.FilterHolder.doFilter(FilterHolder.java:205)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1596)
	at org.apache.solr.servlet.RequiredSolrRequestFilter.doFilter(RequiredSolrRequestFilter.java:79)
	at jakarta.servlet.http.HttpFilter.doFilter(HttpFilter.java:97)
	at org.eclipse.jetty.ee10.servlet.FilterHolder.doFilter(FilterHolder.java:205)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1596)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$MappedServlet.handle(ServletHandler.java:1557)
	at org.eclipse.jetty.ee10.servlet.ServletChannel.dispatch(ServletChannel.java:828)
	at org.eclipse.jetty.ee10.servlet.ServletChannel.handle(ServletChannel.java:438)
	at org.eclipse.jetty.ee10.servlet.ServletHandler.handle(ServletHandler.java:470)
	at org.eclipse.jetty.server.handler.ContextHandler.handle(ContextHandler.java:1102)
	at org.eclipse.jetty.server.Handler$Wrapper.handle(Handler.java:740)
	at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:138)
	at org.eclipse.jetty.server.Handler$Wrapper.handle(Handler.java:740)
	at org.eclipse.jetty.server.handler.GracefulHandler.handle(GracefulHandler.java:112)
	at org.eclipse.jetty.server.Server.handle(Server.java:182)
	at org.eclipse.jetty.server.internal.HttpChannelState$HandlerInvoker.run(HttpChannelState.java:721)
	at org.eclipse.jetty.util.thread.Invocable$ReadyTask.run(Invocable.java:177)
	at org.eclipse.jetty.http2.server.internal.HttpStreamOverHTTP2$1.run(HttpStreamOverHTTP2.java:144)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:981)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.doRunJob(QueuedThreadPool.java:1212)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1166)
	at java.base/java.lang.Thread.run(Thread.java:1527)

>

Stack Trace:
java.lang.AssertionError: ObjectTracker found 3 object(s) that were not released!!! [TransactionLog, TransactionLog, TransactionLog]
org.apache.solr.update.TransactionLog:org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.update.TransactionLog@1666805101
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:57)
	at org.apache.solr.update.TransactionLog.<init>(TransactionLog.java:256)
	at org.apache.solr.update.TransactionLog.<init>(TransactionLog.java:179)
	at org.apache.solr.update.UpdateLog.newTransactionLog(UpdateLog.java:715)
	at org.apache.solr.update.UpdateLog.ensureLog(UpdateLog.java:1677)
	at org.apache.solr.update.UpdateLog.add(UpdateLog.java:820)
	at org.apache.solr.update.UpdateLog.add(UpdateLog.java:800)
	at org.apache.solr.update.DirectUpdateHandler2.doNormalUpdate(DirectUpdateHandler2.java:535)
	at org.apache.solr.update.DirectUpdateHandler2.addDoc0(DirectUpdateHandler2.java:481)
	at org.apache.solr.update.DirectUpdateHandler2.addDoc(DirectUpdateHandler2.java:410)
	at org.apache.solr.update.processor.RunUpdateProcessorFactory$RunUpdateProcessor.processAdd(RunUpdateProcessorFactory.java:76)
	at org.apache.solr.update.processor.UpdateRequestProcessor.processAdd(UpdateRequestProcessor.java:54)
	at org.apache.solr.update.processor.NestedUpdateProcessorFactory$NestedUpdateProcessor.processAdd(NestedUpdateProcessorFactory.java:87)
	at org.apache.solr.update.processor.UpdateRequestProcessor.processAdd(UpdateRequestProcessor.java:54)
	at org.apache.solr.update.processor.DistributedUpdateProcessor.doLocalAdd(DistributedUpdateProcessor.java:274)
	at org.apache.solr.update.processor.DistributedUpdateProcessor.doVersionAdd(DistributedUpdateProcessor.java:520)
	at org.apache.solr.update.processor.DistributedUpdateProcessor.lambda$versionAdd$0(DistributedUpdateProcessor.java:361)
	at org.apache.solr.update.UpdateLocks.runWithLockInternal(UpdateLocks.java:158)
	at org.apache.solr.update.UpdateLocks.runWithLock(UpdateLocks.java:100)
	at org.apache.solr.update.processor.DistributedUpdateProcessor.versionAdd(DistributedUpdateProcessor.java:346)
	at org.apache.solr.update.processor.DistributedUpdateProcessor.processAdd(DistributedUpdateProcessor.java:240)
	at org.apache.solr.update.processor.DistributedZkUpdateProcessor.processAdd(DistributedZkUpdateProcessor.java:280)
	at org.apache.solr.update.processor.LogUpdateProcessorFactory$LogUpdateProcessor.processAdd(LogUpdateProcessorFactory.java:111)
	at org.apache.solr.handler.loader.JavabinLoader$1.update(JavabinLoader.java:122)
	at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$StreamingCodec.readDocs(JavaBinUpdateRequestCodec.java:259)
	at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$StreamingCodec.readIterator(JavaBinUpdateRequestCodec.java:214)
	at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:345)
	at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:290)
	at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$StreamingCodec.readNamedList(JavaBinUpdateRequestCodec.java:200)
	at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:310)
	at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:290)
	at org.apache.solr.common.util.JavaBinCodec.unmarshal(JavaBinCodec.java:199)
	at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$StreamingCodec.unmarshal(JavaBinUpdateRequestCodec.java:182)
	at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.unmarshal(JavaBinUpdateRequestCodec.java:110)
	at org.apache.solr.handler.loader.JavabinLoader.parseAndLoadDocs(JavabinLoader.java:134)
	at org.apache.solr.handler.loader.JavabinLoader.load(JavabinLoader.java:73)
	at org.apache.solr.handler.UpdateRequestHandler$1.load(UpdateRequestHandler.java:101)
	at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:99)
	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:251)
	at org.apache.solr.core.SolrCore.execute(SolrCore.java:2919)
	at org.apache.solr.servlet.HttpSolrCall.executeCoreRequest(HttpSolrCall.java:699)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:492)
	at org.apache.solr.servlet.SolrServlet.dispatch(SolrServlet.java:114)
	at org.apache.solr.servlet.SolrServlet.service(SolrServlet.java:90)
	at jakarta.servlet.http.HttpServlet.service(HttpServlet.java:723)
	at org.eclipse.jetty.ee10.servlet.ServletHolder.handle(ServletHolder.java:753)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1624)
	at org.apache.solr.servlet.AuthenticationFilter.doFilter(AuthenticationFilter.java:79)
	at jakarta.servlet.http.HttpFilter.doFilter(HttpFilter.java:97)
	at org.eclipse.jetty.ee10.servlet.FilterHolder.doFilter(FilterHolder.java:205)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1596)
	at org.apache.solr.servlet.TracingFilter.doFilter(TracingFilter.java:55)
	at jakarta.servlet.http.HttpFilter.doFilter(HttpFilter.java:97)
	at org.eclipse.jetty.ee10.servlet.FilterHolder.doFilter(FilterHolder.java:205)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1596)
	at org.apache.solr.servlet.RateLimitFilter.doFilter(RateLimitFilter.java:50)
	at jakarta.servlet.http.HttpFilter.doFilter(HttpFilter.java:97)
	at org.eclipse.jetty.ee10.servlet.FilterHolder.doFilter(FilterHolder.java:205)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1596)
	at org.apache.solr.servlet.RequiredSolrRequestFilter.doFilter(RequiredSolrRequestFilter.java:79)
	at jakarta.servlet.http.HttpFilter.doFilter(HttpFilter.java:97)
	at org.eclipse.jetty.ee10.servlet.FilterHolder.doFilter(FilterHolder.java:205)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1596)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$MappedServlet.handle(ServletHandler.java:1557)
	at org.eclipse.jetty.ee10.servlet.ServletChannel.dispatch(ServletChannel.java:828)
	at org.eclipse.jetty.ee10.servlet.ServletChannel.handle(ServletChannel.java:438)
	at org.eclipse.jetty.ee10.servlet.ServletHandler.handle(ServletHandler.java:470)
	at org.eclipse.jetty.server.handler.ContextHandler.handle(ContextHandler.java:1102)
	at org.eclipse.jetty.server.Handler$Wrapper.handle(Handler.java:740)
	at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:138)
	at org.eclipse.jetty.server.Handler$Wrapper.handle(Handler.java:740)
	at org.eclipse.jetty.server.handler.GracefulHandler.handle(GracefulHandler.java:112)
	at org.eclipse.jetty.server.Server.handle(Server.java:182)
	at org.eclipse.jetty.server.internal.HttpChannelState$HandlerInvoker.run(HttpChannelState.java:721)
	at org.eclipse.jetty.util.thread.Invocable$ReadyTask.run(Invocable.java:177)
	at org.eclipse.jetty.http2.server.internal.HttpStreamOverHTTP2$1.run(HttpStreamOverHTTP2.java:144)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:981)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.doRunJob(QueuedThreadPool.java:1212)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1166)
	at java.base/java.lang.Thread.run(Thread.java:1527)

org.apache.solr.update.TransactionLog:org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.update.TransactionLog@431332166
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:57)
	at org.apache.solr.update.TransactionLog.<init>(TransactionLog.java:256)
	at org.apache.solr.update.TransactionLog.<init>(TransactionLog.java:179)
	at org.apache.solr.update.UpdateLog.newTransactionLog(UpdateLog.java:715)
	at org.apache.solr.update.UpdateLog.ensureLog(UpdateLog.java:1677)
	at org.apache.solr.update.UpdateLog.add(UpdateLog.java:820)
	at org.apache.solr.update.UpdateLog.add(UpdateLog.java:800)
	at org.apache.solr.update.DirectUpdateHandler2.doNormalUpdate(DirectUpdateHandler2.java:535)
	at org.apache.solr.update.DirectUpdateHandler2.addDoc0(DirectUpdateHandler2.java:481)
	at org.apache.solr.update.DirectUpdateHandler2.addDoc(DirectUpdateHandler2.java:410)
	at org.apache.solr.update.processor.RunUpdateProcessorFactory$RunUpdateProcessor.processAdd(RunUpdateProcessorFactory.java:76)
	at org.apache.solr.update.processor.UpdateRequestProcessor.processAdd(UpdateRequestProcessor.java:54)
	at org.apache.solr.update.processor.NestedUpdateProcessorFactory$NestedUpdateProcessor.processAdd(NestedUpdateProcessorFactory.java:87)
	at org.apache.solr.update.processor.UpdateRequestProcessor.processAdd(UpdateRequestProcessor.java:54)
	at org.apache.solr.update.processor.DistributedUpdateProcessor.doLocalAdd(DistributedUpdateProcessor.java:274)
	at org.apache.solr.update.processor.DistributedUpdateProcessor.doVersionAdd(DistributedUpdateProcessor.java:520)
	at org.apache.solr.update.processor.DistributedUpdateProcessor.lambda$versionAdd$0(DistributedUpdateProcessor.java:361)
	at org.apache.solr.update.UpdateLocks.runWithLockInternal(UpdateLocks.java:158)
	at org.apache.solr.update.UpdateLocks.runWithLock(UpdateLocks.java:100)
	at org.apache.solr.update.processor.DistributedUpdateProcessor.versionAdd(DistributedUpdateProcessor.java:346)
	at org.apache.solr.update.processor.DistributedUpdateProcessor.processAdd(DistributedUpdateProcessor.java:240)
	at org.apache.solr.update.processor.DistributedZkUpdateProcessor.processAdd(DistributedZkUpdateProcessor.java:280)
	at org.apache.solr.update.processor.LogUpdateProcessorFactory$LogUpdateProcessor.processAdd(LogUpdateProcessorFactory.java:111)
	at org.apache.solr.handler.loader.JavabinLoader$1.update(JavabinLoader.java:122)
	at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$StreamingCodec.readDocs(JavaBinUpdateRequestCodec.java:259)
	at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$StreamingCodec.readIterator(JavaBinUpdateRequestCodec.java:214)
	at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:345)
	at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:290)
	at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$StreamingCodec.readNamedList(JavaBinUpdateRequestCodec.java:200)
	at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:310)
	at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:290)
	at org.apache.solr.common.util.JavaBinCodec.unmarshal(JavaBinCodec.java:199)
	at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$StreamingCodec.unmarshal(JavaBinUpdateRequestCodec.java:182)
	at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.unmarshal(JavaBinUpdateRequestCodec.java:110)
	at org.apache.solr.handler.loader.JavabinLoader.parseAndLoadDocs(JavabinLoader.java:134)
	at org.apache.solr.handler.loader.JavabinLoader.load(JavabinLoader.java:73)
	at org.apache.solr.handler.UpdateRequestHandler$1.load(UpdateRequestHandler.java:101)
	at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:99)
	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:251)
	at org.apache.solr.core.SolrCore.execute(SolrCore.java:2919)
	at org.apache.solr.servlet.HttpSolrCall.executeCoreRequest(HttpSolrCall.java:699)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:492)
	at org.apache.solr.servlet.SolrServlet.dispatch(SolrServlet.java:114)
	at org.apache.solr.servlet.SolrServlet.service(SolrServlet.java:90)
	at jakarta.servlet.http.HttpServlet.service(HttpServlet.java:723)
	at org.eclipse.jetty.ee10.servlet.ServletHolder.handle(ServletHolder.java:753)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1624)
	at org.apache.solr.servlet.AuthenticationFilter.doFilter(AuthenticationFilter.java:79)
	at jakarta.servlet.http.HttpFilter.doFilter(HttpFilter.java:97)
	at org.eclipse.jetty.ee10.servlet.FilterHolder.doFilter(FilterHolder.java:205)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1596)
	at org.apache.solr.servlet.TracingFilter.doFilter(TracingFilter.java:55)
	at jakarta.servlet.http.HttpFilter.doFilter(HttpFilter.java:97)
	at org.eclipse.jetty.ee10.servlet.FilterHolder.doFilter(FilterHolder.java:205)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1596)
	at org.apache.solr.servlet.RateLimitFilter.doFilter(RateLimitFilter.java:50)
	at jakarta.servlet.http.HttpFilter.doFilter(HttpFilter.java:97)
	at org.eclipse.jetty.ee10.servlet.FilterHolder.doFilter(FilterHolder.java:205)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1596)
	at org.apache.solr.servlet.RequiredSolrRequestFilter.doFilter(RequiredSolrRequestFilter.java:79)
	at jakarta.servlet.http.HttpFilter.doFilter(HttpFilter.java:97)
	at org.eclipse.jetty.ee10.servlet.FilterHolder.doFilter(FilterHolder.java:205)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1596)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$MappedServlet.handle(ServletHandler.java:1557)
	at org.eclipse.jetty.ee10.servlet.ServletChannel.dispatch(ServletChannel.java:828)
	at org.eclipse.jetty.ee10.servlet.ServletChannel.handle(ServletChannel.java:438)
	at org.eclipse.jetty.ee10.servlet.ServletHandler.handle(ServletHandler.java:470)
	at org.eclipse.jetty.server.handler.ContextHandler.handle(ContextHandler.java:1102)
	at org.eclipse.jetty.server.Handler$Wrapper.handle(Handler.java:740)
	at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:138)
	at org.eclipse.jetty.server.Handler$Wrapper.handle(Handler.java:740)
	at org.eclipse.jetty.server.handler.GracefulHandler.handle(GracefulHandler.java:112)
	at org.eclipse.jetty.server.Server.handle(Server.java:182)
	at org.eclipse.jetty.server.internal.HttpChannelState$HandlerInvoker.run(HttpChannelState.java:721)
	at org.eclipse.jetty.util.thread.Invocable$ReadyTask.run(Invocable.java:177)
	at org.eclipse.jetty.http2.server.internal.HttpStreamOverHTTP2$1.run(HttpStreamOverHTTP2.java:144)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:981)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.doRunJob(QueuedThreadPool.java:1212)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1166)
	at java.base/java.lang.Thread.run(Thread.java:1527)

org.apache.solr.update.TransactionLog:org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.update.TransactionLog@1098408656
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:57)
	at org.apache.solr.update.TransactionLog.<init>(TransactionLog.java:256)
	at org.apache.solr.update.TransactionLog.<init>(TransactionLog.java:179)
	at org.apache.solr.update.UpdateLog.newTransactionLog(UpdateLog.java:715)
	at org.apache.solr.update.UpdateLog.ensureLog(UpdateLog.java:1677)
	at org.apache.solr.update.UpdateLog.delete(UpdateLog.java:886)
	at org.apache.solr.update.DirectUpdateHandler2.delete(DirectUpdateHandler2.java:623)
	at org.apache.solr.update.processor.RunUpdateProcessorFactory$RunUpdateProcessor.processDelete(RunUpdateProcessorFactory.java:84)
	at org.apache.solr.update.processor.UpdateRequestProcessor.processDelete(UpdateRequestProcessor.java:58)
	at org.apache.solr.update.processor.UpdateRequestProcessor.processDelete(UpdateRequestProcessor.java:58)
	at org.apache.solr.update.processor.DistributedUpdateProcessor.doLocalDelete(DistributedUpdateProcessor.java:280)
	at org.apache.solr.update.processor.DistributedUpdateProcessor.doVersionDelete(DistributedUpdateProcessor.java:1140)
	at org.apache.solr.update.processor.DistributedUpdateProcessor.lambda$versionDelete$0(DistributedUpdateProcessor.java:1044)
	at org.apache.solr.update.UpdateLocks.runWithLockInternal(UpdateLocks.java:158)
	at org.apache.solr.update.UpdateLocks.runWithLock(UpdateLocks.java:100)
	at org.apache.solr.update.processor.DistributedUpdateProcessor.versionDelete(DistributedUpdateProcessor.java:1041)
	at org.apache.solr.update.processor.DistributedUpdateProcessor.doDeleteById(DistributedUpdateProcessor.java:811)
	at org.apache.solr.update.processor.DistributedZkUpdateProcessor.doDeleteById(DistributedZkUpdateProcessor.java:376)
	at org.apache.solr.update.processor.DistributedUpdateProcessor.processDelete(DistributedUpdateProcessor.java:797)
	at org.apache.solr.update.processor.DistributedZkUpdateProcessor.processDelete(DistributedZkUpdateProcessor.java:353)
	at org.apache.solr.update.processor.LogUpdateProcessorFactory$LogUpdateProcessor.processDelete(LogUpdateProcessorFactory.java:134)
	at org.apache.solr.handler.loader.JavabinLoader.delete(JavabinLoader.java:220)
	at org.apache.solr.handler.loader.JavabinLoader.parseAndLoadDocs(JavabinLoader.java:139)
	at org.apache.solr.handler.loader.JavabinLoader.load(JavabinLoader.java:73)
	at org.apache.solr.handler.UpdateRequestHandler$1.load(UpdateRequestHandler.java:101)
	at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:99)
	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:251)
	at org.apache.solr.core.SolrCore.execute(SolrCore.java:2919)
	at org.apache.solr.servlet.HttpSolrCall.executeCoreRequest(HttpSolrCall.java:699)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:492)
	at org.apache.solr.servlet.SolrServlet.dispatch(SolrServlet.java:114)
	at org.apache.solr.servlet.SolrServlet.service(SolrServlet.java:90)
	at jakarta.servlet.http.HttpServlet.service(HttpServlet.java:723)
	at org.eclipse.jetty.ee10.servlet.ServletHolder.handle(ServletHolder.java:753)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1624)
	at org.apache.solr.servlet.AuthenticationFilter.doFilter(AuthenticationFilter.java:79)
	at jakarta.servlet.http.HttpFilter.doFilter(HttpFilter.java:97)
	at org.eclipse.jetty.ee10.servlet.FilterHolder.doFilter(FilterHolder.java:205)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1596)
	at org.apache.solr.servlet.TracingFilter.doFilter(TracingFilter.java:55)
	at jakarta.servlet.http.HttpFilter.doFilter(HttpFilter.java:97)
	at org.eclipse.jetty.ee10.servlet.FilterHolder.doFilter(FilterHolder.java:205)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1596)
	at org.apache.solr.servlet.RateLimitFilter.doFilter(RateLimitFilter.java:50)
	at jakarta.servlet.http.HttpFilter.doFilter(HttpFilter.java:97)
	at org.eclipse.jetty.ee10.servlet.FilterHolder.doFilter(FilterHolder.java:205)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1596)
	at org.apache.solr.servlet.RequiredSolrRequestFilter.doFilter(RequiredSolrRequestFilter.java:79)
	at jakarta.servlet.http.HttpFilter.doFilter(HttpFilter.java:97)
	at org.eclipse.jetty.ee10.servlet.FilterHolder.doFilter(FilterHolder.java:205)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1596)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$MappedServlet.handle(ServletHandler.java:1557)
	at org.eclipse.jetty.ee10.servlet.ServletChannel.dispatch(ServletChannel.java:828)
	at org.eclipse.jetty.ee10.servlet.ServletChannel.handle(ServletChannel.java:438)
	at org.eclipse.jetty.ee10.servlet.ServletHandler.handle(ServletHandler.java:470)
	at org.eclipse.jetty.server.handler.ContextHandler.handle(ContextHandler.java:1102)
	at org.eclipse.jetty.server.Handler$Wrapper.handle(Handler.java:740)
	at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:138)
	at org.eclipse.jetty.server.Handler$Wrapper.handle(Handler.java:740)
	at org.eclipse.jetty.server.handler.GracefulHandler.handle(GracefulHandler.java:112)
	at org.eclipse.jetty.server.Server.handle(Server.java:182)
	at org.eclipse.jetty.server.internal.HttpChannelState$HandlerInvoker.run(HttpChannelState.java:721)
	at org.eclipse.jetty.util.thread.Invocable$ReadyTask.run(Invocable.java:177)
	at org.eclipse.jetty.http2.server.internal.HttpStreamOverHTTP2$1.run(HttpStreamOverHTTP2.java:144)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:981)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.doRunJob(QueuedThreadPool.java:1212)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1166)
	at java.base/java.lang.Thread.run(Thread.java:1527)

 expected null, but was:<ObjectTracker found 3 object(s) that were not released!!! [TransactionLog, TransactionLog, TransactionLog]
org.apache.solr.update.TransactionLog:org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.update.TransactionLog@1666805101
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:57)
	at org.apache.solr.update.TransactionLog.<init>(TransactionLog.java:256)
	at org.apache.solr.update.TransactionLog.<init>(TransactionLog.java:179)
	at org.apache.solr.update.UpdateLog.newTransactionLog(UpdateLog.java:715)
	at org.apache.solr.update.UpdateLog.ensureLog(UpdateLog.java:1677)
	at org.apache.solr.update.UpdateLog.add(UpdateLog.java:820)
	at org.apache.solr.update.UpdateLog.add(UpdateLog.java:800)
	at org.apache.solr.update.DirectUpdateHandler2.doNormalUpdate(DirectUpdateHandler2.java:535)
	at org.apache.solr.update.DirectUpdateHandler2.addDoc0(DirectUpdateHandler2.java:481)
	at org.apache.solr.update.DirectUpdateHandler2.addDoc(DirectUpdateHandler2.java:410)
	at org.apache.solr.update.processor.RunUpdateProcessorFactory$RunUpdateProcessor.processAdd(RunUpdateProcessorFactory.java:76)
	at org.apache.solr.update.processor.UpdateRequestProcessor.processAdd(UpdateRequestProcessor.java:54)
	at org.apache.solr.update.processor.NestedUpdateProcessorFactory$NestedUpdateProcessor.processAdd(NestedUpdateProcessorFactory.java:87)
	at org.apache.solr.update.processor.UpdateRequestProcessor.processAdd(UpdateRequestProcessor.java:54)
	at org.apache.solr.update.processor.DistributedUpdateProcessor.doLocalAdd(DistributedUpdateProcessor.java:274)
	at org.apache.solr.update.processor.DistributedUpdateProcessor.doVersionAdd(DistributedUpdateProcessor.java:520)
	at org.apache.solr.update.processor.DistributedUpdateProcessor.lambda$versionAdd$0(DistributedUpdateProcessor.java:361)
	at org.apache.solr.update.UpdateLocks.runWithLockInternal(UpdateLocks.java:158)
	at org.apache.solr.update.UpdateLocks.runWithLock(UpdateLocks.java:100)
	at org.apache.solr.update.processor.DistributedUpdateProcessor.versionAdd(DistributedUpdateProcessor.java:346)
	at org.apache.solr.update.processor.DistributedUpdateProcessor.processAdd(DistributedUpdateProcessor.java:240)
	at org.apache.solr.update.processor.DistributedZkUpdateProcessor.processAdd(DistributedZkUpdateProcessor.java:280)
	at org.apache.solr.update.processor.LogUpdateProcessorFactory$LogUpdateProcessor.processAdd(LogUpdateProcessorFactory.java:111)
	at org.apache.solr.handler.loader.JavabinLoader$1.update(JavabinLoader.java:122)
	at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$StreamingCodec.readDocs(JavaBinUpdateRequestCodec.java:259)
	at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$StreamingCodec.readIterator(JavaBinUpdateRequestCodec.java:214)
	at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:345)
	at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:290)
	at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$StreamingCodec.readNamedList(JavaBinUpdateRequestCodec.java:200)
	at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:310)
	at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:290)
	at org.apache.solr.common.util.JavaBinCodec.unmarshal(JavaBinCodec.java:199)
	at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$StreamingCodec.unmarshal(JavaBinUpdateRequestCodec.java:182)
	at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.unmarshal(JavaBinUpdateRequestCodec.java:110)
	at org.apache.solr.handler.loader.JavabinLoader.parseAndLoadDocs(JavabinLoader.java:134)
	at org.apache.solr.handler.loader.JavabinLoader.load(JavabinLoader.java:73)
	at org.apache.solr.handler.UpdateRequestHandler$1.load(UpdateRequestHandler.java:101)
	at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:99)
	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:251)
	at org.apache.solr.core.SolrCore.execute(SolrCore.java:2919)
	at org.apache.solr.servlet.HttpSolrCall.executeCoreRequest(HttpSolrCall.java:699)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:492)
	at org.apache.solr.servlet.SolrServlet.dispatch(SolrServlet.java:114)
	at org.apache.solr.servlet.SolrServlet.service(SolrServlet.java:90)
	at jakarta.servlet.http.HttpServlet.service(HttpServlet.java:723)
	at org.eclipse.jetty.ee10.servlet.ServletHolder.handle(ServletHolder.java:753)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1624)
	at org.apache.solr.servlet.AuthenticationFilter.doFilter(AuthenticationFilter.java:79)
	at jakarta.servlet.http.HttpFilter.doFilter(HttpFilter.java:97)
	at org.eclipse.jetty.ee10.servlet.FilterHolder.doFilter(FilterHolder.java:205)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1596)
	at org.apache.solr.servlet.TracingFilter.doFilter(TracingFilter.java:55)
	at jakarta.servlet.http.HttpFilter.doFilter(HttpFilter.java:97)
	at org.eclipse.jetty.ee10.servlet.FilterHolder.doFilter(FilterHolder.java:205)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1596)
	at org.apache.solr.servlet.RateLimitFilter.doFilter(RateLimitFilter.java:50)
	at jakarta.servlet.http.HttpFilter.doFilter(HttpFilter.java:97)
	at org.eclipse.jetty.ee10.servlet.FilterHolder.doFilter(FilterHolder.java:205)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1596)
	at org.apache.solr.servlet.RequiredSolrRequestFilter.doFilter(RequiredSolrRequestFilter.java:79)
	at jakarta.servlet.http.HttpFilter.doFilter(HttpFilter.java:97)
	at org.eclipse.jetty.ee10.servlet.FilterHolder.doFilter(FilterHolder.java:205)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1596)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$MappedServlet.handle(ServletHandler.java:1557)
	at org.eclipse.jetty.ee10.servlet.ServletChannel.dispatch(ServletChannel.java:828)
	at org.eclipse.jetty.ee10.servlet.ServletChannel.handle(ServletChannel.java:438)
	at org.eclipse.jetty.ee10.servlet.ServletHandler.handle(ServletHandler.java:470)
	at org.eclipse.jetty.server.handler.ContextHandler.handle(ContextHandler.java:1102)
	at org.eclipse.jetty.server.Handler$Wrapper.handle(Handler.java:740)
	at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:138)
	at org.eclipse.jetty.server.Handler$Wrapper.handle(Handler.java:740)
	at org.eclipse.jetty.server.handler.GracefulHandler.handle(GracefulHandler.java:112)
	at org.eclipse.jetty.server.Server.handle(Server.java:182)
	at org.eclipse.jetty.server.internal.HttpChannelState$HandlerInvoker.run(HttpChannelState.java:721)
	at org.eclipse.jetty.util.thread.Invocable$ReadyTask.run(Invocable.java:177)
	at org.eclipse.jetty.http2.server.internal.HttpStreamOverHTTP2$1.run(HttpStreamOverHTTP2.java:144)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:981)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.doRunJob(QueuedThreadPool.java:1212)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1166)
	at java.base/java.lang.Thread.run(Thread.java:1527)

org.apache.solr.update.TransactionLog:org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.update.TransactionLog@431332166
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:57)
	at org.apache.solr.update.TransactionLog.<init>(TransactionLog.java:256)
	at org.apache.solr.update.TransactionLog.<init>(TransactionLog.java:179)
	at org.apache.solr.update.UpdateLog.newTransactionLog(UpdateLog.java:715)
	at org.apache.solr.update.UpdateLog.ensureLog(UpdateLog.java:1677)
	at org.apache.solr.update.UpdateLog.add(UpdateLog.java:820)
	at org.apache.solr.update.UpdateLog.add(UpdateLog.java:800)
	at org.apache.solr.update.DirectUpdateHandler2.doNormalUpdate(DirectUpdateHandler2.java:535)
	at org.apache.solr.update.DirectUpdateHandler2.addDoc0(DirectUpdateHandler2.java:481)
	at org.apache.solr.update.DirectUpdateHandler2.addDoc(DirectUpdateHandler2.java:410)
	at org.apache.solr.update.processor.RunUpdateProcessorFactory$RunUpdateProcessor.processAdd(RunUpdateProcessorFactory.java:76)
	at org.apache.solr.update.processor.UpdateRequestProcessor.processAdd(UpdateRequestProcessor.java:54)
	at org.apache.solr.update.processor.NestedUpdateProcessorFactory$NestedUpdateProcessor.processAdd(NestedUpdateProcessorFactory.java:87)
	at org.apache.solr.update.processor.UpdateRequestProcessor.processAdd(UpdateRequestProcessor.java:54)
	at org.apache.solr.update.processor.DistributedUpdateProcessor.doLocalAdd(DistributedUpdateProcessor.java:274)
	at org.apache.solr.update.processor.DistributedUpdateProcessor.doVersionAdd(DistributedUpdateProcessor.java:520)
	at org.apache.solr.update.processor.DistributedUpdateProcessor.lambda$versionAdd$0(DistributedUpdateProcessor.java:361)
	at org.apache.solr.update.UpdateLocks.runWithLockInternal(UpdateLocks.java:158)
	at org.apache.solr.update.UpdateLocks.runWithLock(UpdateLocks.java:100)
	at org.apache.solr.update.processor.DistributedUpdateProcessor.versionAdd(DistributedUpdateProcessor.java:346)
	at org.apache.solr.update.processor.DistributedUpdateProcessor.processAdd(DistributedUpdateProcessor.java:240)
	at org.apache.solr.update.processor.DistributedZkUpdateProcessor.processAdd(DistributedZkUpdateProcessor.java:280)
	at org.apache.solr.update.processor.LogUpdateProcessorFactory$LogUpdateProcessor.processAdd(LogUpdateProcessorFactory.java:111)
	at org.apache.solr.handler.loader.JavabinLoader$1.update(JavabinLoader.java:122)
	at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$StreamingCodec.readDocs(JavaBinUpdateRequestCodec.java:259)
	at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$StreamingCodec.readIterator(JavaBinUpdateRequestCodec.java:214)
	at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:345)
	at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:290)
	at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$StreamingCodec.readNamedList(JavaBinUpdateRequestCodec.java:200)
	at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:310)
	at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:290)
	at org.apache.solr.common.util.JavaBinCodec.unmarshal(JavaBinCodec.java:199)
	at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$StreamingCodec.unmarshal(JavaBinUpdateRequestCodec.java:182)
	at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.unmarshal(JavaBinUpdateRequestCodec.java:110)
	at org.apache.solr.handler.loader.JavabinLoader.parseAndLoadDocs(JavabinLoader.java:134)
	at org.apache.solr.handler.loader.JavabinLoader.load(JavabinLoader.java:73)
	at org.apache.solr.handler.UpdateRequestHandler$1.load(UpdateRequestHandler.java:101)
	at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:99)
	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:251)
	at org.apache.solr.core.SolrCore.execute(SolrCore.java:2919)
	at org.apache.solr.servlet.HttpSolrCall.executeCoreRequest(HttpSolrCall.java:699)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:492)
	at org.apache.solr.servlet.SolrServlet.dispatch(SolrServlet.java:114)
	at org.apache.solr.servlet.SolrServlet.service(SolrServlet.java:90)
	at jakarta.servlet.http.HttpServlet.service(HttpServlet.java:723)
	at org.eclipse.jetty.ee10.servlet.ServletHolder.handle(ServletHolder.java:753)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1624)
	at org.apache.solr.servlet.AuthenticationFilter.doFilter(AuthenticationFilter.java:79)
	at jakarta.servlet.http.HttpFilter.doFilter(HttpFilter.java:97)
	at org.eclipse.jetty.ee10.servlet.FilterHolder.doFilter(FilterHolder.java:205)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1596)
	at org.apache.solr.servlet.TracingFilter.doFilter(TracingFilter.java:55)
	at jakarta.servlet.http.HttpFilter.doFilter(HttpFilter.java:97)
	at org.eclipse.jetty.ee10.servlet.FilterHolder.doFilter(FilterHolder.java:205)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1596)
	at org.apache.solr.servlet.RateLimitFilter.doFilter(RateLimitFilter.java:50)
	at jakarta.servlet.http.HttpFilter.doFilter(HttpFilter.java:97)
	at org.eclipse.jetty.ee10.servlet.FilterHolder.doFilter(FilterHolder.java:205)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1596)
	at org.apache.solr.servlet.RequiredSolrRequestFilter.doFilter(RequiredSolrRequestFilter.java:79)
	at jakarta.servlet.http.HttpFilter.doFilter(HttpFilter.java:97)
	at org.eclipse.jetty.ee10.servlet.FilterHolder.doFilter(FilterHolder.java:205)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1596)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$MappedServlet.handle(ServletHandler.java:1557)
	at org.eclipse.jetty.ee10.servlet.ServletChannel.dispatch(ServletChannel.java:828)
	at org.eclipse.jetty.ee10.servlet.ServletChannel.handle(ServletChannel.java:438)
	at org.eclipse.jetty.ee10.servlet.ServletHandler.handle(ServletHandler.java:470)
	at org.eclipse.jetty.server.handler.ContextHandler.handle(ContextHandler.java:1102)
	at org.eclipse.jetty.server.Handler$Wrapper.handle(Handler.java:740)
	at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:138)
	at org.eclipse.jetty.server.Handler$Wrapper.handle(Handler.java:740)
	at org.eclipse.jetty.server.handler.GracefulHandler.handle(GracefulHandler.java:112)
	at org.eclipse.jetty.server.Server.handle(Server.java:182)
	at org.eclipse.jetty.server.internal.HttpChannelState$HandlerInvoker.run(HttpChannelState.java:721)
	at org.eclipse.jetty.util.thread.Invocable$ReadyTask.run(Invocable.java:177)
	at org.eclipse.jetty.http2.server.internal.HttpStreamOverHTTP2$1.run(HttpStreamOverHTTP2.java:144)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:981)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.doRunJob(QueuedThreadPool.java:1212)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1166)
	at java.base/java.lang.Thread.run(Thread.java:1527)

org.apache.solr.update.TransactionLog:org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.update.TransactionLog@1098408656
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:57)
	at org.apache.solr.update.TransactionLog.<init>(TransactionLog.java:256)
	at org.apache.solr.update.TransactionLog.<init>(TransactionLog.java:179)
	at org.apache.solr.update.UpdateLog.newTransactionLog(UpdateLog.java:715)
	at org.apache.solr.update.UpdateLog.ensureLog(UpdateLog.java:1677)
	at org.apache.solr.update.UpdateLog.delete(UpdateLog.java:886)
	at org.apache.solr.update.DirectUpdateHandler2.delete(DirectUpdateHandler2.java:623)
	at org.apache.solr.update.processor.RunUpdateProcessorFactory$RunUpdateProcessor.processDelete(RunUpdateProcessorFactory.java:84)
	at org.apache.solr.update.processor.UpdateRequestProcessor.processDelete(UpdateRequestProcessor.java:58)
	at org.apache.solr.update.processor.UpdateRequestProcessor.processDelete(UpdateRequestProcessor.java:58)
	at org.apache.solr.update.processor.DistributedUpdateProcessor.doLocalDelete(DistributedUpdateProcessor.java:280)
	at org.apache.solr.update.processor.DistributedUpdateProcessor.doVersionDelete(DistributedUpdateProcessor.java:1140)
	at org.apache.solr.update.processor.DistributedUpdateProcessor.lambda$versionDelete$0(DistributedUpdateProcessor.java:1044)
	at org.apache.solr.update.UpdateLocks.runWithLockInternal(UpdateLocks.java:158)
	at org.apache.solr.update.UpdateLocks.runWithLock(UpdateLocks.java:100)
	at org.apache.solr.update.processor.DistributedUpdateProcessor.versionDelete(DistributedUpdateProcessor.java:1041)
	at org.apache.solr.update.processor.DistributedUpdateProcessor.doDeleteById(DistributedUpdateProcessor.java:811)
	at org.apache.solr.update.processor.DistributedZkUpdateProcessor.doDeleteById(DistributedZkUpdateProcessor.java:376)
	at org.apache.solr.update.processor.DistributedUpdateProcessor.processDelete(DistributedUpdateProcessor.java:797)
	at org.apache.solr.update.processor.DistributedZkUpdateProcessor.processDelete(DistributedZkUpdateProcessor.java:353)
	at org.apache.solr.update.processor.LogUpdateProcessorFactory$LogUpdateProcessor.processDelete(LogUpdateProcessorFactory.java:134)
	at org.apache.solr.handler.loader.JavabinLoader.delete(JavabinLoader.java:220)
	at org.apache.solr.handler.loader.JavabinLoader.parseAndLoadDocs(JavabinLoader.java:139)
	at org.apache.solr.handler.loader.JavabinLoader.load(JavabinLoader.java:73)
	at org.apache.solr.handler.UpdateRequestHandler$1.load(UpdateRequestHandler.java:101)
	at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:99)
	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:251)
	at org.apache.solr.core.SolrCore.execute(SolrCore.java:2919)
	at org.apache.solr.servlet.HttpSolrCall.executeCoreRequest(HttpSolrCall.java:699)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:492)
	at org.apache.solr.servlet.SolrServlet.dispatch(SolrServlet.java:114)
	at org.apache.solr.servlet.SolrServlet.service(SolrServlet.java:90)
	at jakarta.servlet.http.HttpServlet.service(HttpServlet.java:723)
	at org.eclipse.jetty.ee10.servlet.ServletHolder.handle(ServletHolder.java:753)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1624)
	at org.apache.solr.servlet.AuthenticationFilter.doFilter(AuthenticationFilter.java:79)
	at jakarta.servlet.http.HttpFilter.doFilter(HttpFilter.java:97)
	at org.eclipse.jetty.ee10.servlet.FilterHolder.doFilter(FilterHolder.java:205)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1596)
	at org.apache.solr.servlet.TracingFilter.doFilter(TracingFilter.java:55)
	at jakarta.servlet.http.HttpFilter.doFilter(HttpFilter.java:97)
	at org.eclipse.jetty.ee10.servlet.FilterHolder.doFilter(FilterHolder.java:205)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1596)
	at org.apache.solr.servlet.RateLimitFilter.doFilter(RateLimitFilter.java:50)
	at jakarta.servlet.http.HttpFilter.doFilter(HttpFilter.java:97)
	at org.eclipse.jetty.ee10.servlet.FilterHolder.doFilter(FilterHolder.java:205)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1596)
	at org.apache.solr.servlet.RequiredSolrRequestFilter.doFilter(RequiredSolrRequestFilter.java:79)
	at jakarta.servlet.http.HttpFilter.doFilter(HttpFilter.java:97)
	at org.eclipse.jetty.ee10.servlet.FilterHolder.doFilter(FilterHolder.java:205)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1596)
	at org.eclipse.jetty.ee10.servlet.ServletHandler$MappedServlet.handle(ServletHandler.java:1557)
	at org.eclipse.jetty.ee10.servlet.ServletChannel.dispatch(ServletChannel.java:828)
	at org.eclipse.jetty.ee10.servlet.ServletChannel.handle(ServletChannel.java:438)
	at org.eclipse.jetty.ee10.servlet.ServletHandler.handle(ServletHandler.java:470)
	at org.eclipse.jetty.server.handler.ContextHandler.handle(ContextHandler.java:1102)
	at org.eclipse.jetty.server.Handler$Wrapper.handle(Handler.java:740)
	at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:138)
	at org.eclipse.jetty.server.Handler$Wrapper.handle(Handler.java:740)
	at org.eclipse.jetty.server.handler.GracefulHandler.handle(GracefulHandler.java:112)
	at org.eclipse.jetty.server.Server.handle(Server.java:182)
	at org.eclipse.jetty.server.internal.HttpChannelState$HandlerInvoker.run(HttpChannelState.java:721)
	at org.eclipse.jetty.util.thread.Invocable$ReadyTask.run(Invocable.java:177)
	at org.eclipse.jetty.http2.server.internal.HttpStreamOverHTTP2$1.run(HttpStreamOverHTTP2.java:144)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:981)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.doRunJob(QueuedThreadPool.java:1212)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1166)
	at java.base/java.lang.Thread.run(Thread.java:1527)

>
	at __randomizedtesting.SeedInfo.seed([EE2824C348162770]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.junit.Assert.failNotNull(Assert.java:756)
	at org.junit.Assert.assertNull(Assert.java:738)
	at org.apache.solr.SolrTestCase$1.afterIfSuccessful(SolrTestCase.java:100)
	at com.carrotsearch.randomizedtesting.rules.TestRuleAdapter$1.afterIfSuccessful(TestRuleAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:37)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
	at java.base/java.lang.Thread.run(Thread.java:1527)


FAILED:  org.apache.solr.cloud.BasicDistributedZkTest.test

Error Message:
java.lang.RuntimeException: Timeout waiting for pool to shutdown. [poolName=closeThreadPool][poolName=closeThreadPool] org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor@549a7c34[Shutting down, pool size = 2, active threads = 2, queued tasks = 0, completed tasks = 8]

Stack Trace:
java.lang.RuntimeException: Timeout waiting for pool to shutdown. [poolName=closeThreadPool][poolName=closeThreadPool] org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor@549a7c34[Shutting down, pool size = 2, active threads = 2, queued tasks = 0, completed tasks = 8]
	at __randomizedtesting.SeedInfo.seed([EE2824C348162770:667C1B19E6EA4A88]:0)
	at org.apache.solr.common.util.ExecutorUtil.awaitTermination(ExecutorUtil.java:167)
	at org.apache.solr.common.util.ExecutorUtil.awaitTermination(ExecutorUtil.java:153)
	at org.apache.solr.common.util.ExecutorUtil.shutdownAndAwaitTermination(ExecutorUtil.java:119)
	at org.apache.solr.BaseDistributedSearchTestCase.destroyServers(BaseDistributedSearchTestCase.java:371)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.destroyServers(AbstractFullDistribZkTestBase.java:2269)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1065)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1034)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
	at java.base/java.lang.Thread.run(Thread.java:1527)


FAILED:  org.apache.solr.s3.S3InstallShardTest.classMethod

Error Message:
java.lang.Exception: Error shutting down MiniSolrCloudCluster

Stack Trace:
java.lang.Exception: Error shutting down MiniSolrCloudCluster
	at __randomizedtesting.SeedInfo.seed([EE2824C348162770]:0)
	at org.apache.solr.cloud.MiniSolrCloudCluster.checkForExceptions(MiniSolrCloudCluster.java:748)
	at org.apache.solr.cloud.MiniSolrCloudCluster.shutdown(MiniSolrCloudCluster.java:669)
	at org.apache.solr.cloud.SolrCloudTestCase.shutdownCluster(SolrCloudTestCase.java:144)
	at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
	at java.base/java.lang.reflect.Method.invoke(Method.java:565)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1763)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:909)
	at com.adobe.testing.s3mock.junit4.S3MockRule$1.evaluate(S3MockRule.java:70)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
	at java.base/java.lang.Thread.run(Thread.java:1527)
	Suppressed: java.lang.RuntimeException: Timeout waiting for pool to shutdown. [poolName=closeThreadPool][poolName=closeThreadPool] org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor@54c2313c[Shutting down, pool size = 1, active threads = 1, queued tasks = 0, completed tasks = 4]
		at org.apache.solr.common.util.ExecutorUtil.awaitTermination(ExecutorUtil.java:167)
		at org.apache.solr.common.util.ExecutorUtil.awaitTermination(ExecutorUtil.java:153)
		at org.apache.solr.common.util.ExecutorUtil.shutdownAndAwaitTermination(ExecutorUtil.java:119)
		at org.apache.solr.core.CoreContainer.shutdown(CoreContainer.java:1325)
		at org.apache.solr.servlet.CoreContainerProvider.close(CoreContainerProvider.java:132)
		at org.apache.solr.servlet.CoreContainerProvider.contextDestroyed(CoreContainerProvider.java:90)
		at org.eclipse.jetty.ee10.servlet.ServletContextHandler.callContextDestroyed(ServletContextHandler.java:1608)
		at org.eclipse.jetty.ee10.servlet.ServletContextHandler.contextDestroyed(ServletContextHandler.java:529)
		at org.eclipse.jetty.ee10.servlet.ServletHandler.doStop(ServletHandler.java:329)
		at org.eclipse.jetty.util.component.AbstractLifeCycle.stop(AbstractLifeCycle.java:132)
		at org.eclipse.jetty.util.component.ContainerLifeCycle.stop(ContainerLifeCycle.java:180)
		at org.eclipse.jetty.util.component.ContainerLifeCycle.doStop(ContainerLifeCycle.java:202)
		at org.eclipse.jetty.server.Handler$Abstract.doStop(Handler.java:499)
		at org.eclipse.jetty.server.handler.ContextHandler.lambda$doStop$0(ContextHandler.java:841)
		at org.eclipse.jetty.server.handler.ContextHandler$ScopedContext.call(ContextHandler.java:1488)
		at org.eclipse.jetty.server.handler.ContextHandler.doStop(ContextHandler.java:841)
		at org.eclipse.jetty.ee10.servlet.ServletContextHandler.stopContext(ServletContextHandler.java:1334)
		at org.eclipse.jetty.ee10.servlet.ServletContextHandler.doStop(ServletContextHandler.java:1085)
		at org.eclipse.jetty.util.component.AbstractLifeCycle.stop(AbstractLifeCycle.java:132)
		at org.eclipse.jetty.util.component.ContainerLifeCycle.stop(ContainerLifeCycle.java:180)
		at org.eclipse.jetty.util.component.ContainerLifeCycle.doStop(ContainerLifeCycle.java:202)
		at org.eclipse.jetty.server.Handler$Abstract.doStop(Handler.java:499)
		at org.eclipse.jetty.util.component.AbstractLifeCycle.stop(AbstractLifeCycle.java:132)
		at org.eclipse.jetty.util.component.ContainerLifeCycle.stop(ContainerLifeCycle.java:180)
		at org.eclipse.jetty.util.component.ContainerLifeCycle.doStop(ContainerLifeCycle.java:202)
		at org.eclipse.jetty.server.Handler$Abstract.doStop(Handler.java:499)
		at org.eclipse.jetty.util.component.AbstractLifeCycle.stop(AbstractLifeCycle.java:132)
		at org.eclipse.jetty.util.component.ContainerLifeCycle.stop(ContainerLifeCycle.java:180)
		at org.eclipse.jetty.util.component.ContainerLifeCycle.doStop(ContainerLifeCycle.java:202)
		at org.eclipse.jetty.server.Handler$Abstract.doStop(Handler.java:499)
		at org.eclipse.jetty.server.Server.doStop(Server.java:690)
		at org.eclipse.jetty.util.component.AbstractLifeCycle.stop(AbstractLifeCycle.java:132)
		at org.apache.solr.embedded.JettySolrRunner.stop(JettySolrRunner.java:586)
		at org.apache.solr.cloud.MiniSolrCloudCluster.stopJettySolrRunner(MiniSolrCloudCluster.java:561)
		at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$shutdown$1(MiniSolrCloudCluster.java:661)
		at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:330)
		at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:408)
		at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1090)
		at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
		... 1 more