You are viewing a plain text version of this content. The canonical link for it is here.
Posted to builds@solr.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2022/02/16 20:41:19 UTC

[JENKINS] Solr-main-MacOSX (64bit/jdk-16.0.2) - Build # 653 - Still unstable!

Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/653/
Java: 64bit/jdk-16.0.2 -XX:+UseCompressedOops -XX:+UseShenandoahGC

12 tests failed.
FAILED:  org.apache.solr.cloud.MultiThreadedOCPTest.test

Error Message:
java.lang.AssertionError: Long running task finished too early, can't test expected null, but was:<1645042297882>

Stack Trace:
java.lang.AssertionError: Long running task finished too early, can't test expected null, but was:<1645042297882>
	at __randomizedtesting.SeedInfo.seed([82F630980F99949A:AA20F42A165F962]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.junit.Assert.failNotNull(Assert.java:756)
	at org.junit.Assert.assertNull(Assert.java:738)
	at org.apache.solr.cloud.MultiThreadedOCPTest.testFillWorkQueue(MultiThreadedOCPTest.java:87)
	at org.apache.solr.cloud.MultiThreadedOCPTest.test(MultiThreadedOCPTest.java:61)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:78)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:567)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1081)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1052)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:831)


FAILED:  org.apache.solr.handler.TestSolrConfigHandlerCloud.test

Error Message:
java.lang.AssertionError: {
  "responseHeader":{
    "status":500,
    "QTime":30031},
  "errorMessages":["4 out of 5 the property overlay to be of version 4 within 30 seconds! Failed cores: [https://127.0.0.1:53552/collection1_shard2_replica_n1/, https://127.0.0.1:53585/collection1_shard2_replica_n6/, https://127.0.0.1:53572/collection1_shard1_replica_n2/, https://127.0.0.1:53592/collection1_shard1_replica_n5/]\n"],
  "WARNING":"This response format is experimental.  It is likely to change in the future.",
  "error":{
    "metadata":[
      "error-class","org.apache.solr.common.SolrException",
      "root-error-class","org.apache.solr.common.SolrException"],
    "msg":"4 out of 5 the property overlay to be of version 4 within 30 seconds! Failed cores: [https://127.0.0.1:53552/collection1_shard2_replica_n1/, https://127.0.0.1:53585/collection1_shard2_replica_n6/, https://127.0.0.1:53572/collection1_shard1_replica_n2/, https://127.0.0.1:53592/collection1_shard1_replica_n5/]",
    "trace":"org.apache.solr.common.SolrException: 4 out of 5 the property overlay to be of version 4 within 30 seconds! Failed cores: [https://127.0.0.1:53552/collection1_shard2_replica_n1/, https://127.0.0.1:53585/collection1_shard2_replica_n6/, https://127.0.0.1:53572/collection1_shard1_replica_n2/, https://127.0.0.1:53592/collection1_shard1_replica_n5/]\n\tat org.apache.solr.handler.SolrConfigHandler.waitForAllReplicasState(SolrConfigHandler.java:812)\n\tat org.apache.solr.handler.SolrConfigHandler$Command.handleCommands(SolrConfigHandler.java:534)\n\tat org.apache.solr.handler.SolrConfigHandler$Command.handlePOST(SolrConfigHandler.java:366)\n\tat org.apache.solr.handler.SolrConfigHandler.handleRequestBody(SolrConfigHandler.java:137)\n\tat org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:214)\n\tat org.apache.solr.core.SolrCore.execute(SolrCore.java:2638)\n\tat org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:824)\n\tat org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:564)\n\tat org.apache.solr.servlet.SolrDispatchFilter.dispatch(SolrDispatchFilter.java:218)\n\tat org.apache.solr.servlet.SolrDispatchFilter.lambda$doFilter$0(SolrDispatchFilter.java:189)\n\tat org.apache.solr.servlet.ServletUtils.traceHttpRequestExecution2(ServletUtils.java:232)\n\tat org.apache.solr.servlet.ServletUtils.rateLimitRequest(ServletUtils.java:208)\n\tat org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:187)\n\tat org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:171)\n\tat org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)\n\tat org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)\n\tat org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:189)\n\tat org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)\n\tat org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)\n\tat org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)\n\tat org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)\n\tat org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1434)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)\n\tat org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)\n\tat org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)\n\tat org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1349)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)\n\tat org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)\n\tat org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)\n\tat org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:713)\n\tat org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)\n\tat org.eclipse.jetty.server.Server.handle(Server.java:516)\n\tat org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:400)\n\tat org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:645)\n\tat org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:392)\n\tat org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:277)\n\tat org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311)\n\tat org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:105)\n\tat org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:555)\n\tat org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:410)\n\tat org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:164)\n\tat org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:105)\n\tat org.eclipse.jetty.io.ChannelEndPoint$1.run(ChannelEndPoint.java:104)\n\tat org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)\n\tat org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)\n\tat java.base/java.lang.Thread.run(Thread.java:831)\n",
    "code":500}}
 expected null, but was:<[4 out of 5 the property overlay to be of version 4 within 30 seconds! Failed cores: [https://127.0.0.1:53552/collection1_shard2_replica_n1/, https://127.0.0.1:53585/collection1_shard2_replica_n6/, https://127.0.0.1:53572/collection1_shard1_replica_n2/, https://127.0.0.1:53592/collection1_shard1_replica_n5/]
]>

Stack Trace:
java.lang.AssertionError: {
  "responseHeader":{
    "status":500,
    "QTime":30031},
  "errorMessages":["4 out of 5 the property overlay to be of version 4 within 30 seconds! Failed cores: [https://127.0.0.1:53552/collection1_shard2_replica_n1/, https://127.0.0.1:53585/collection1_shard2_replica_n6/, https://127.0.0.1:53572/collection1_shard1_replica_n2/, https://127.0.0.1:53592/collection1_shard1_replica_n5/]\n"],
  "WARNING":"This response format is experimental.  It is likely to change in the future.",
  "error":{
    "metadata":[
      "error-class","org.apache.solr.common.SolrException",
      "root-error-class","org.apache.solr.common.SolrException"],
    "msg":"4 out of 5 the property overlay to be of version 4 within 30 seconds! Failed cores: [https://127.0.0.1:53552/collection1_shard2_replica_n1/, https://127.0.0.1:53585/collection1_shard2_replica_n6/, https://127.0.0.1:53572/collection1_shard1_replica_n2/, https://127.0.0.1:53592/collection1_shard1_replica_n5/]",
    "trace":"org.apache.solr.common.SolrException: 4 out of 5 the property overlay to be of version 4 within 30 seconds! Failed cores: [https://127.0.0.1:53552/collection1_shard2_replica_n1/, https://127.0.0.1:53585/collection1_shard2_replica_n6/, https://127.0.0.1:53572/collection1_shard1_replica_n2/, https://127.0.0.1:53592/collection1_shard1_replica_n5/]\n\tat org.apache.solr.handler.SolrConfigHandler.waitForAllReplicasState(SolrConfigHandler.java:812)\n\tat org.apache.solr.handler.SolrConfigHandler$Command.handleCommands(SolrConfigHandler.java:534)\n\tat org.apache.solr.handler.SolrConfigHandler$Command.handlePOST(SolrConfigHandler.java:366)\n\tat org.apache.solr.handler.SolrConfigHandler.handleRequestBody(SolrConfigHandler.java:137)\n\tat org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:214)\n\tat org.apache.solr.core.SolrCore.execute(SolrCore.java:2638)\n\tat org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:824)\n\tat org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:564)\n\tat org.apache.solr.servlet.SolrDispatchFilter.dispatch(SolrDispatchFilter.java:218)\n\tat org.apache.solr.servlet.SolrDispatchFilter.lambda$doFilter$0(SolrDispatchFilter.java:189)\n\tat org.apache.solr.servlet.ServletUtils.traceHttpRequestExecution2(ServletUtils.java:232)\n\tat org.apache.solr.servlet.ServletUtils.rateLimitRequest(ServletUtils.java:208)\n\tat org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:187)\n\tat org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:171)\n\tat org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)\n\tat org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)\n\tat org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:189)\n\tat org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)\n\tat org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)\n\tat org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)\n\tat org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)\n\tat org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1434)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)\n\tat org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)\n\tat org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)\n\tat org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1349)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)\n\tat org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)\n\tat org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)\n\tat org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:713)\n\tat org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)\n\tat org.eclipse.jetty.server.Server.handle(Server.java:516)\n\tat org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:400)\n\tat org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:645)\n\tat org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:392)\n\tat org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:277)\n\tat org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311)\n\tat org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:105)\n\tat org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:555)\n\tat org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:410)\n\tat org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:164)\n\tat org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:105)\n\tat org.eclipse.jetty.io.ChannelEndPoint$1.run(ChannelEndPoint.java:104)\n\tat org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)\n\tat org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)\n\tat java.base/java.lang.Thread.run(Thread.java:831)\n",
    "code":500}}
 expected null, but was:<[4 out of 5 the property overlay to be of version 4 within 30 seconds! Failed cores: [https://127.0.0.1:53552/collection1_shard2_replica_n1/, https://127.0.0.1:53585/collection1_shard2_replica_n6/, https://127.0.0.1:53572/collection1_shard1_replica_n2/, https://127.0.0.1:53592/collection1_shard1_replica_n5/]
]>
	at __randomizedtesting.SeedInfo.seed([82F630980F99949A:AA20F42A165F962]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.junit.Assert.failNotNull(Assert.java:756)
	at org.junit.Assert.assertNull(Assert.java:738)
	at org.apache.solr.core.TestSolrConfigHandler.runConfigCommand(TestSolrConfigHandler.java:221)
	at org.apache.solr.core.TestSolrConfigHandler.reqhandlertests(TestSolrConfigHandler.java:321)
	at org.apache.solr.handler.TestSolrConfigHandlerCloud.testReqHandlerAPIs(TestSolrConfigHandlerCloud.java:80)
	at org.apache.solr.handler.TestSolrConfigHandlerCloud.test(TestSolrConfigHandlerCloud.java:50)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:78)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:567)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1081)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1052)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:831)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
java.security.AccessControlException: access denied ("java.io.FilePermission" "/Users/jenkins/workspace/Solr-main-MacOSX/solr/core/build/resources/test/solr/filestore" "write")

Stack Trace:
java.security.AccessControlException: access denied ("java.io.FilePermission" "/Users/jenkins/workspace/Solr-main-MacOSX/solr/core/build/resources/test/solr/filestore" "write")
	at __randomizedtesting.SeedInfo.seed([82F630980F99949A]:0)
	at java.base/java.security.AccessControlContext.checkPermission(AccessControlContext.java:472)
	at java.base/java.security.AccessController.checkPermission(AccessController.java:1036)
	at java.base/java.lang.SecurityManager.checkPermission(SecurityManager.java:408)
	at java.base/java.lang.SecurityManager.checkWrite(SecurityManager.java:838)
	at java.base/sun.nio.fs.UnixPath.checkWrite(UnixPath.java:816)
	at java.base/sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:388)
	at java.base/java.nio.file.Files.createDirectory(Files.java:694)
	at java.base/java.nio.file.Files.createAndCheckIsDirectory(Files.java:801)
	at java.base/java.nio.file.Files.createDirectories(Files.java:747)
	at org.apache.solr.filestore.DistribPackageStore.getPackageStoreDirPath(DistribPackageStore.java:584)
	at org.apache.solr.filestore.DistribPackageStore.ensurePackageStoreDir(DistribPackageStore.java:567)
	at org.apache.solr.filestore.DistribPackageStore.<init>(DistribPackageStore.java:83)
	at org.apache.solr.filestore.PackageStoreAPI.<init>(PackageStoreAPI.java:76)
	at org.apache.solr.core.CoreContainer.load(CoreContainer.java:725)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:184)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:156)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:162)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:112)
	at org.apache.solr.SolrTestCaseJ4.createCore(SolrTestCaseJ4.java:813)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:803)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:650)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:639)
	at org.apache.solr.cloud.AbstractZkTestCase.azt_beforeClass(AbstractZkTestCase.java:68)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:78)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:567)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:882)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:831)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 8 threads leaked from SUITE scope at org.apache.solr.rest.TestManagedResourceStorage: 
   1) Thread[id=22027, name=OverseerCollectionConfigSetProcessor-72069369449021443-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:271)
        at app//org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:258)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   2) Thread[id=22019, name=h2sc-14940-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800ed6c80.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$260/0x0000000800f83168.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   3) Thread[id=22024, name=SUITE-TestManagedResourceStorage-seed#[82F630980F99949A]-worker-EventThread, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:551)
   4) Thread[id=22028, name=zkCallback-14944-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   5) Thread[id=22026, name=OverseerStateUpdate-72069369449021443-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:294)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   6) Thread[id=22016, name=httpShardExecutor-14938-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800ed6c80.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$260/0x0000000800f83168.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   7) Thread[id=22025, name=zkConnectionManagerCallback-14945-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   8) Thread[id=22023, name=SUITE-TestManagedResourceStorage-seed#[82F630980F99949A]-worker-SendThread(127.0.0.1:52145), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:214)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1380)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanAndNotifyState(ClientCnxn.java:1321)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1299)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 8 threads leaked from SUITE scope at org.apache.solr.rest.TestManagedResourceStorage: 
   1) Thread[id=22027, name=OverseerCollectionConfigSetProcessor-72069369449021443-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:271)
        at app//org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:258)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   2) Thread[id=22019, name=h2sc-14940-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800ed6c80.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$260/0x0000000800f83168.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   3) Thread[id=22024, name=SUITE-TestManagedResourceStorage-seed#[82F630980F99949A]-worker-EventThread, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:551)
   4) Thread[id=22028, name=zkCallback-14944-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   5) Thread[id=22026, name=OverseerStateUpdate-72069369449021443-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:294)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   6) Thread[id=22016, name=httpShardExecutor-14938-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800ed6c80.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$260/0x0000000800f83168.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   7) Thread[id=22025, name=zkConnectionManagerCallback-14945-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   8) Thread[id=22023, name=SUITE-TestManagedResourceStorage-seed#[82F630980F99949A]-worker-SendThread(127.0.0.1:52145), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:214)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1380)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanAndNotifyState(ClientCnxn.java:1321)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1299)
	at __randomizedtesting.SeedInfo.seed([82F630980F99949A]:0)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=22019, name=h2sc-14940-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800ed6c80.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$260/0x0000000800f83168.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   2) Thread[id=22028, name=zkCallback-14944-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   3) Thread[id=22016, name=httpShardExecutor-14938-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800ed6c80.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$260/0x0000000800f83168.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   4) Thread[id=22025, name=zkConnectionManagerCallback-14945-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   5) Thread[id=22023, name=SUITE-TestManagedResourceStorage-seed#[82F630980F99949A]-worker-SendThread(127.0.0.1:52145), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1136)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1200)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=22019, name=h2sc-14940-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800ed6c80.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$260/0x0000000800f83168.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   2) Thread[id=22028, name=zkCallback-14944-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   3) Thread[id=22016, name=httpShardExecutor-14938-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800ed6c80.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$260/0x0000000800f83168.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   4) Thread[id=22025, name=zkConnectionManagerCallback-14945-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   5) Thread[id=22023, name=SUITE-TestManagedResourceStorage-seed#[82F630980F99949A]-worker-SendThread(127.0.0.1:52145), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1136)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1200)
	at __randomizedtesting.SeedInfo.seed([82F630980F99949A]:0)


FAILED:  org.apache.solr.schema.PreAnalyzedFieldManagedSchemaCloudTest.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=15936, name=qtp1191041164-15936, state=RUNNABLE, group=TGRP-PreAnalyzedFieldManagedSchemaCloudTest], registration stack trace below.

Stack Trace:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=15936, name=qtp1191041164-15936, state=RUNNABLE, group=TGRP-PreAnalyzedFieldManagedSchemaCloudTest], registration stack trace below.
	at java.base@16.0.2/java.lang.Thread.getStackTrace(Thread.java:1596)
	at app//com.carrotsearch.randomizedtesting.RandomizedContext.closeAtEnd(RandomizedContext.java:157)
	at app//org.apache.lucene.util.LuceneTestCase.closeAfterSuite(LuceneTestCase.java:798)
	at app//org.apache.lucene.util.LuceneTestCase.wrapDirectory(LuceneTestCase.java:1504)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1368)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1350)
	at app//org.apache.solr.core.MockDirectoryFactory.create(MockDirectoryFactory.java:49)
	at app//org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:360)
	at app//org.apache.solr.core.SolrCore.initSnapshotMetaDataManager(SolrCore.java:582)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1026)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:961)
	at app//org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1423)
	at app//org.apache.solr.core.CoreContainer.create(CoreContainer.java:1297)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:345)
	at app//org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:423)
	at app//org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:197)
	at app//org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:214)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:866)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:830)
	at app//org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:542)
	at app//org.apache.solr.servlet.SolrDispatchFilter.dispatch(SolrDispatchFilter.java:218)
	at app//org.apache.solr.servlet.SolrDispatchFilter.lambda$doFilter$0(SolrDispatchFilter.java:189)
	at app//org.apache.solr.servlet.ServletUtils.traceHttpRequestExecution2(ServletUtils.java:232)
	at app//org.apache.solr.servlet.ServletUtils.rateLimitRequest(ServletUtils.java:208)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:187)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:171)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:189)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1434)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
	at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)
	at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1349)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
	at app//org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:713)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.server.Server.handle(Server.java:516)
	at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:400)
	at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:645)
	at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:392)
	at app//org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:352)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
	at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
Caused by: java.lang.AssertionError: Directory not closed: MockDirectoryWrapper(ByteBuffersDirectory@1bd9438f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@673d3ee8)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.lucene.util.CloseableDirectory.close(CloseableDirectory.java:43)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:788)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:785)
	at com.carrotsearch.randomizedtesting.RandomizedContext.closeResources(RandomizedContext.java:225)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.afterAlways(RandomizedRunner.java:801)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	... 1 more


FAILED:  org.apache.solr.schema.PreAnalyzedFieldManagedSchemaCloudTest.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=15936, name=qtp1191041164-15936, state=RUNNABLE, group=TGRP-PreAnalyzedFieldManagedSchemaCloudTest], registration stack trace below.

Stack Trace:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=15936, name=qtp1191041164-15936, state=RUNNABLE, group=TGRP-PreAnalyzedFieldManagedSchemaCloudTest], registration stack trace below.
	at java.base@16.0.2/java.lang.Thread.getStackTrace(Thread.java:1596)
	at app//com.carrotsearch.randomizedtesting.RandomizedContext.closeAtEnd(RandomizedContext.java:157)
	at app//org.apache.lucene.util.LuceneTestCase.closeAfterSuite(LuceneTestCase.java:798)
	at app//org.apache.lucene.util.LuceneTestCase.wrapDirectory(LuceneTestCase.java:1504)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1368)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1350)
	at app//org.apache.solr.core.MockDirectoryFactory.create(MockDirectoryFactory.java:49)
	at app//org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:360)
	at app//org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:404)
	at app//org.apache.solr.core.SolrCore.initIndex(SolrCore.java:817)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1030)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:961)
	at app//org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1423)
	at app//org.apache.solr.core.CoreContainer.create(CoreContainer.java:1297)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:345)
	at app//org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:423)
	at app//org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:197)
	at app//org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:214)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:866)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:830)
	at app//org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:542)
	at app//org.apache.solr.servlet.SolrDispatchFilter.dispatch(SolrDispatchFilter.java:218)
	at app//org.apache.solr.servlet.SolrDispatchFilter.lambda$doFilter$0(SolrDispatchFilter.java:189)
	at app//org.apache.solr.servlet.ServletUtils.traceHttpRequestExecution2(ServletUtils.java:232)
	at app//org.apache.solr.servlet.ServletUtils.rateLimitRequest(ServletUtils.java:208)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:187)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:171)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:189)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1434)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
	at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)
	at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1349)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
	at app//org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:713)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.server.Server.handle(Server.java:516)
	at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:400)
	at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:645)
	at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:392)
	at app//org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:352)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
	at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
Caused by: java.lang.AssertionError: Directory not closed: MockDirectoryWrapper(ByteBuffersDirectory@5d4eb32a lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6bba6da0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.lucene.util.CloseableDirectory.close(CloseableDirectory.java:43)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:788)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:785)
	at com.carrotsearch.randomizedtesting.RandomizedContext.closeResources(RandomizedContext.java:225)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.afterAlways(RandomizedRunner.java:801)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	... 1 more


FAILED:  org.apache.solr.schema.PreAnalyzedFieldManagedSchemaCloudTest.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=15936, name=qtp1191041164-15936, state=RUNNABLE, group=TGRP-PreAnalyzedFieldManagedSchemaCloudTest], registration stack trace below.

Stack Trace:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=15936, name=qtp1191041164-15936, state=RUNNABLE, group=TGRP-PreAnalyzedFieldManagedSchemaCloudTest], registration stack trace below.
	at java.base@16.0.2/java.lang.Thread.getStackTrace(Thread.java:1596)
	at app//com.carrotsearch.randomizedtesting.RandomizedContext.closeAtEnd(RandomizedContext.java:157)
	at app//org.apache.lucene.util.LuceneTestCase.closeAfterSuite(LuceneTestCase.java:798)
	at app//org.apache.lucene.util.LuceneTestCase.wrapDirectory(LuceneTestCase.java:1504)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1368)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1350)
	at app//org.apache.solr.core.MockDirectoryFactory.create(MockDirectoryFactory.java:49)
	at app//org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:360)
	at app//org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:98)
	at app//org.apache.solr.core.SolrCore.initIndex(SolrCore.java:849)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1030)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:961)
	at app//org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1423)
	at app//org.apache.solr.core.CoreContainer.create(CoreContainer.java:1297)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:345)
	at app//org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:423)
	at app//org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:197)
	at app//org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:214)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:866)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:830)
	at app//org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:542)
	at app//org.apache.solr.servlet.SolrDispatchFilter.dispatch(SolrDispatchFilter.java:218)
	at app//org.apache.solr.servlet.SolrDispatchFilter.lambda$doFilter$0(SolrDispatchFilter.java:189)
	at app//org.apache.solr.servlet.ServletUtils.traceHttpRequestExecution2(ServletUtils.java:232)
	at app//org.apache.solr.servlet.ServletUtils.rateLimitRequest(ServletUtils.java:208)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:187)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:171)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:189)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1434)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
	at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)
	at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1349)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
	at app//org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:713)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.server.Server.handle(Server.java:516)
	at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:400)
	at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:645)
	at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:392)
	at app//org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:352)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
	at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
Caused by: java.lang.AssertionError: Directory not closed: MockDirectoryWrapper(ByteBuffersDirectory@274f53f1 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1ccb9359)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.lucene.util.CloseableDirectory.close(CloseableDirectory.java:43)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:788)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:785)
	at com.carrotsearch.randomizedtesting.RandomizedContext.closeResources(RandomizedContext.java:225)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.afterAlways(RandomizedRunner.java:801)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	... 1 more


FAILED:  org.apache.solr.schema.PreAnalyzedFieldManagedSchemaCloudTest.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.schema.PreAnalyzedFieldManagedSchemaCloudTest: 
   1) Thread[id=16020, name=searcherExecutor-8708-thread-1-processing-127.0.0.1:60092_solr, state=WAITING, group=TGRP-PreAnalyzedFieldManagedSchemaCloudTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:714)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1046)
        at java.base@16.0.2/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:232)
        at app//org.apache.solr.core.SolrCore.lambda$new$2(SolrCore.java:1048)
        at app//org.apache.solr.core.SolrCore$$Lambda$724/0x0000000801312ec8.call(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.FutureTask.run(FutureTask.java:264)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$241/0x0000000800f216b0.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   2) Thread[id=15998, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-PreAnalyzedFieldManagedSchemaCloudTest]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:156)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:89)
        at app//org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:294)
        at app//org.apache.solr.schema.ZkIndexSchemaReader.createSchemaWatcher(ZkIndexSchemaReader.java:95)
        at app//org.apache.solr.schema.ZkIndexSchemaReader.<init>(ZkIndexSchemaReader.java:76)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.inform(ManagedIndexSchemaFactory.java:478)
        at app//org.apache.solr.core.SolrResourceLoader.inform(SolrResourceLoader.java:688)
        at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1061)
        at app//org.apache.solr.core.SolrCore.reload(SolrCore.java:744)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1690)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3134)
        at app//org.apache.solr.core.SolrCore$$Lambda$750/0x000000080132acd8.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2499)
        at app//org.apache.solr.cloud.ZkController$$Lambda$871/0x000000080137d798.run(Unknown Source)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.schema.PreAnalyzedFieldManagedSchemaCloudTest: 
   1) Thread[id=16020, name=searcherExecutor-8708-thread-1-processing-127.0.0.1:60092_solr, state=WAITING, group=TGRP-PreAnalyzedFieldManagedSchemaCloudTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:714)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1046)
        at java.base@16.0.2/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:232)
        at app//org.apache.solr.core.SolrCore.lambda$new$2(SolrCore.java:1048)
        at app//org.apache.solr.core.SolrCore$$Lambda$724/0x0000000801312ec8.call(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.FutureTask.run(FutureTask.java:264)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$241/0x0000000800f216b0.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   2) Thread[id=15998, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-PreAnalyzedFieldManagedSchemaCloudTest]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:156)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:89)
        at app//org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:294)
        at app//org.apache.solr.schema.ZkIndexSchemaReader.createSchemaWatcher(ZkIndexSchemaReader.java:95)
        at app//org.apache.solr.schema.ZkIndexSchemaReader.<init>(ZkIndexSchemaReader.java:76)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.inform(ManagedIndexSchemaFactory.java:478)
        at app//org.apache.solr.core.SolrResourceLoader.inform(SolrResourceLoader.java:688)
        at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1061)
        at app//org.apache.solr.core.SolrCore.reload(SolrCore.java:744)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1690)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3134)
        at app//org.apache.solr.core.SolrCore$$Lambda$750/0x000000080132acd8.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2499)
        at app//org.apache.solr.cloud.ZkController$$Lambda$871/0x000000080137d798.run(Unknown Source)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
	at __randomizedtesting.SeedInfo.seed([82F630980F99949A]:0)


FAILED:  org.apache.solr.search.facet.TestCloudJSONFacetSKGEquiv.testRandom

Error Message:
java.lang.AssertionError: rows=0&q=*:*&fore=*:*&back=(field_1_multi_sds:29+OR+field_12_multi_ss:50+OR+field_8_multi_sdsS:29)&json.facet={"facet_1":{"method":"${method_val:smart}","limit":-1,"overrequest":3,"prefix":"3","sort":"index+asc","refine":false,"type":"terms","field":"field_7_solo_sd","facet":{"skg":{"type":"func","func":"relatedness($fore,$back)","${sweep_key:xxx}":"${sweep_val:yyy}"},"facet_2":{"method":"${method_val:smart}","limit":39,"overrequest":0,"allBuckets":false,"refine":true,"type":"terms","field":"field_9_solo_s","facet":{"skg":{"type":"func","func":"relatedness($fore,$back)","${sweep_key:xxx}":"${sweep_val:yyy}"},"skg0":{"min_popularity":"0.001","type":"func","func":"relatedness($fore,$back)","${sweep_key:xxx}":"${sweep_val:yyy}"},"skg1":{"type":"func","func":"relatedness({!v%3D'(field_2_multi_sdsS:23+OR+field_9_multi_ss:12+OR+field_14_multi_sdsS:56)'},{!v%3D'(field_6_multi_ss:26+OR+field_2_multi_sdsS:7+OR+field_13_multi_sds:14+OR+field_12_multi_ss:53+OR+field_2_multi_sdsS:27+OR+field_1_multi_sds:36+OR+field_4_multi_sds:50+OR+field_5_multi_sdsS:28+OR+field_11_multi_sdsS:20)'})","${sweep_key:xxx}":"${sweep_val:yyy}"}}},"sum":"sum(field_6_solo_i)"}},"facet_3":{"method":"${method_val:smart}","limit":-1,"overrequest":0,"sort":"index+desc","allBuckets":true,"type":"terms","field":"field_14_multi_sdsS","facet":{"skg":{"type":"func","func":"relatedness($fore,$back)","${sweep_key:xxx}":"${sweep_val:yyy}"}}}} ===> Mismatch: .facet_1.buckets.length:6!=4 using method_val=enum&sweep_key=sweep_collection&sweep_val=true

Stack Trace:
java.lang.AssertionError: rows=0&q=*:*&fore=*:*&back=(field_1_multi_sds:29+OR+field_12_multi_ss:50+OR+field_8_multi_sdsS:29)&json.facet={"facet_1":{"method":"${method_val:smart}","limit":-1,"overrequest":3,"prefix":"3","sort":"index+asc","refine":false,"type":"terms","field":"field_7_solo_sd","facet":{"skg":{"type":"func","func":"relatedness($fore,$back)","${sweep_key:xxx}":"${sweep_val:yyy}"},"facet_2":{"method":"${method_val:smart}","limit":39,"overrequest":0,"allBuckets":false,"refine":true,"type":"terms","field":"field_9_solo_s","facet":{"skg":{"type":"func","func":"relatedness($fore,$back)","${sweep_key:xxx}":"${sweep_val:yyy}"},"skg0":{"min_popularity":"0.001","type":"func","func":"relatedness($fore,$back)","${sweep_key:xxx}":"${sweep_val:yyy}"},"skg1":{"type":"func","func":"relatedness({!v%3D'(field_2_multi_sdsS:23+OR+field_9_multi_ss:12+OR+field_14_multi_sdsS:56)'},{!v%3D'(field_6_multi_ss:26+OR+field_2_multi_sdsS:7+OR+field_13_multi_sds:14+OR+field_12_multi_ss:53+OR+field_2_multi_sdsS:27+OR+field_1_multi_sds:36+OR+field_4_multi_sds:50+OR+field_5_multi_sdsS:28+OR+field_11_multi_sdsS:20)'})","${sweep_key:xxx}":"${sweep_val:yyy}"}}},"sum":"sum(field_6_solo_i)"}},"facet_3":{"method":"${method_val:smart}","limit":-1,"overrequest":0,"sort":"index+desc","allBuckets":true,"type":"terms","field":"field_14_multi_sdsS","facet":{"skg":{"type":"func","func":"relatedness($fore,$back)","${sweep_key:xxx}":"${sweep_val:yyy}"}}}} ===> Mismatch: .facet_1.buckets.length:6!=4 using method_val=enum&sweep_key=sweep_collection&sweep_val=true
	at __randomizedtesting.SeedInfo.seed([82F630980F99949A:F0BA1597BEF922E9]:0)
	at org.apache.solr.search.facet.TestCloudJSONFacetSKGEquiv.assertFacetSKGsAreConsistent(TestCloudJSONFacetSKGEquiv.java:757)
	at org.apache.solr.search.facet.TestCloudJSONFacetSKGEquiv.testRandom(TestCloudJSONFacetSKGEquiv.java:673)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:78)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:567)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:831)
Caused by: java.lang.AssertionError: Mismatch: .facet_1.buckets.length:6!=4 using method_val=enum&sweep_key=sweep_collection&sweep_val=true
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.solr.search.facet.TestCloudJSONFacetSKGEquiv.assertFacetSKGsAreConsistent(TestCloudJSONFacetSKGEquiv.java:752)
	... 47 more


FAILED:  org.apache.solr.core.TestLazyCores.testDontEvictUsedCore

Error Message:
java.lang.AssertionError: long request should still be busy

Stack Trace:
java.lang.AssertionError: long request should still be busy
	at __randomizedtesting.SeedInfo.seed([82F630980F99949A:F565F141542CCC1E]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.junit.Assert.assertTrue(Assert.java:42)
	at org.apache.solr.core.TestLazyCores.testDontEvictUsedCore(TestLazyCores.java:940)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:78)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:567)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:831)
	Suppressed: junit.framework.AssertionFailedError: LogListener-INFO-1 recorded log events which were not expected & removed from the queue by the test code expected:<0> but was:<1>
		at junit.framework.Assert.fail(Assert.java:57)
		at junit.framework.Assert.failNotEquals(Assert.java:329)
		at junit.framework.Assert.assertEquals(Assert.java:78)
		at junit.framework.Assert.assertEquals(Assert.java:234)
		at org.apache.solr.util.LogListener.close(LogListener.java:230)
		at org.apache.solr.core.TestLazyCores.testDontEvictUsedCore(TestLazyCores.java:909)
		... 46 more


FAILED:  org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=6839, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:156)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:89)
        at app//org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:306)
        at app//org.apache.solr.cloud.ZkController.pathExists(ZkController.java:1139)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:80)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getSnowballWordSet(AbstractAnalysisFactory.java:321)
        at app//org.apache.lucene.analysis.en.AbstractWordsFileFilterFactory.inform(AbstractWordsFileFilterFactory.java:90)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:723)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1341)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1194)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1179)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:615)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:177)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:107)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:244)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:314)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:241)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$627/0x00000008011d2c08.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:46)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:239)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1686)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3134)
        at app//org.apache.solr.core.SolrCore$$Lambda$750/0x000000080132acd8.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2499)
        at app//org.apache.solr.cloud.ZkController$$Lambda$871/0x000000080137d798.run(Unknown Source)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=6839, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:156)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:89)
        at app//org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:306)
        at app//org.apache.solr.cloud.ZkController.pathExists(ZkController.java:1139)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:80)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getSnowballWordSet(AbstractAnalysisFactory.java:321)
        at app//org.apache.lucene.analysis.en.AbstractWordsFileFilterFactory.inform(AbstractWordsFileFilterFactory.java:90)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:723)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1341)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1194)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1179)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:615)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:177)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:107)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:244)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:314)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:241)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$627/0x00000008011d2c08.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:46)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:239)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1686)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3134)
        at app//org.apache.solr.core.SolrCore$$Lambda$750/0x000000080132acd8.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2499)
        at app//org.apache.solr.cloud.ZkController$$Lambda$871/0x000000080137d798.run(Unknown Source)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
	at __randomizedtesting.SeedInfo.seed([82F630980F99949A]:0)

[JENKINS] Solr-main-MacOSX (64bit/jdk-16.0.2) - Build # 678 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/678/
Java: 64bit/jdk-16.0.2 -XX:-UseCompressedOops -XX:+UseParallelGC

7 tests failed.
FAILED:  org.apache.solr.cloud.OverseerModifyCollectionTest.testModifyColl

Error Message:
java.lang.AssertionError: expected:<FAILED> but was:<COMPLETED>

Stack Trace:
java.lang.AssertionError: expected:<FAILED> but was:<COMPLETED>
	at __randomizedtesting.SeedInfo.seed([1A5FAFA6CE7F7FF0:B3C785D3C2B3BBF6]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.junit.Assert.failNotEquals(Assert.java:835)
	at org.junit.Assert.assertEquals(Assert.java:120)
	at org.junit.Assert.assertEquals(Assert.java:146)
	at org.apache.solr.cloud.OverseerModifyCollectionTest.testModifyColl(OverseerModifyCollectionTest.java:51)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:78)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:567)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:831)


FAILED:  org.apache.solr.schema.TestBulkSchemaConcurrent.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 6 threads leaked from SUITE scope at org.apache.solr.schema.TestBulkSchemaConcurrent: 
   1) Thread[id=11255, name=ZKEventListenerThread, state=WAITING, group=TGRP-TestBulkSchemaConcurrent]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:714)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireShared(AbstractQueuedSynchronizer.java:1026)
        at java.base@16.0.2/java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.lock(ReentrantReadWriteLock.java:738)
        at java.management@16.0.2/com.sun.jmx.mbeanserver.Repository.retrieve(Repository.java:484)
        at java.management@16.0.2/com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.getMBean(DefaultMBeanServerInterceptor.java:1077)
        at java.management@16.0.2/com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.getAttribute(DefaultMBeanServerInterceptor.java:637)
        at java.management@16.0.2/com.sun.jmx.mbeanserver.JmxMBeanServer.getAttribute(JmxMBeanServer.java:678)
        at java.management@16.0.2/javax.management.AttributeValueExp.getAttribute(AttributeValueExp.java:162)
        at java.management@16.0.2/javax.management.AttributeValueExp.apply(AttributeValueExp.java:104)
        at java.management@16.0.2/javax.management.BinaryRelQueryExp.apply(BinaryRelQueryExp.java:109)
        at java.management@16.0.2/com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.filterListOfObjectInstances(DefaultMBeanServerInterceptor.java:1599)
        at java.management@16.0.2/com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.queryMBeans(DefaultMBeanServerInterceptor.java:495)
        at java.management@16.0.2/com.sun.jmx.mbeanserver.JmxMBeanServer.queryMBeans(JmxMBeanServer.java:592)
        at app//org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter$JmxListener.unregisterMBean(JmxMetricsReporter.java:555)
        at app//org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter$JmxListener.unregisterAll(JmxMetricsReporter.java:709)
        at app//org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter.close(JmxMetricsReporter.java:759)
        at app//org.apache.solr.metrics.reporters.SolrJmxReporter.close(SolrJmxReporter.java:129)
        at app//org.apache.solr.metrics.SolrMetricManager.lambda$closeReporters$5(SolrMetricManager.java:1157)
        at app//org.apache.solr.metrics.SolrMetricManager$$Lambda$961/0x000000080133f9e8.accept(Unknown Source)
        at java.base@16.0.2/java.lang.Iterable.forEach(Iterable.java:75)
        at app//org.apache.solr.metrics.SolrMetricManager.closeReporters(SolrMetricManager.java:1151)
        at app//org.apache.solr.metrics.SolrCoreMetricManager.close(SolrCoreMetricManager.java:145)
        at app//org.apache.solr.core.SolrCore.doClose(SolrCore.java:1596)
        at app//org.apache.solr.core.SolrCore.close(SolrCore.java:1582)
        at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1100)
        at app//org.apache.solr.core.SolrCore.reload(SolrCore.java:744)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1686)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3134)
        at app//org.apache.solr.core.SolrCore$$Lambda$756/0x0000000801221330.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2499)
        at app//org.apache.solr.cloud.ZkController$$Lambda$951/0x000000080133e028.run(Unknown Source)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   2) Thread[id=11229, name=ZKEventListenerThread, state=WAITING, group=TGRP-TestBulkSchemaConcurrent]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:714)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireShared(AbstractQueuedSynchronizer.java:1026)
        at java.base@16.0.2/java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.lock(ReentrantReadWriteLock.java:738)
        at java.management@16.0.2/com.sun.jmx.mbeanserver.Repository.query(Repository.java:525)
        at java.management@16.0.2/com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.queryMBeansImpl(DefaultMBeanServerInterceptor.java:507)
        at java.management@16.0.2/com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.queryMBeans(DefaultMBeanServerInterceptor.java:476)
        at java.management@16.0.2/com.sun.jmx.mbeanserver.JmxMBeanServer.queryMBeans(JmxMBeanServer.java:592)
        at app//org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter$JmxListener.unregisterMBean(JmxMetricsReporter.java:555)
        at app//org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter$JmxListener.unregisterAll(JmxMetricsReporter.java:709)
        at app//org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter.close(JmxMetricsReporter.java:759)
        at app//org.apache.solr.metrics.reporters.SolrJmxReporter.close(SolrJmxReporter.java:129)
        at app//org.apache.solr.metrics.SolrMetricManager.lambda$closeReporters$5(SolrMetricManager.java:1157)
        at app//org.apache.solr.metrics.SolrMetricManager$$Lambda$961/0x000000080133f9e8.accept(Unknown Source)
        at java.base@16.0.2/java.lang.Iterable.forEach(Iterable.java:75)
        at app//org.apache.solr.metrics.SolrMetricManager.closeReporters(SolrMetricManager.java:1151)
        at app//org.apache.solr.metrics.SolrCoreMetricManager.close(SolrCoreMetricManager.java:145)
        at app//org.apache.solr.core.SolrCore.doClose(SolrCore.java:1596)
        at app//org.apache.solr.core.SolrCore.close(SolrCore.java:1582)
        at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1100)
        at app//org.apache.solr.core.SolrCore.reload(SolrCore.java:744)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1686)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3134)
        at app//org.apache.solr.core.SolrCore$$Lambda$756/0x0000000801221330.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2499)
        at app//org.apache.solr.cloud.ZkController$$Lambda$951/0x000000080133e028.run(Unknown Source)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   3) Thread[id=11252, name=ZKEventListenerThread, state=WAITING, group=TGRP-TestBulkSchemaConcurrent]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:714)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireShared(AbstractQueuedSynchronizer.java:1026)
        at java.base@16.0.2/java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.lock(ReentrantReadWriteLock.java:738)
        at java.management@16.0.2/com.sun.jmx.mbeanserver.Repository.query(Repository.java:525)
        at java.management@16.0.2/com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.queryMBeansImpl(DefaultMBeanServerInterceptor.java:507)
        at java.management@16.0.2/com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.queryMBeans(DefaultMBeanServerInterceptor.java:476)
        at java.management@16.0.2/com.sun.jmx.mbeanserver.JmxMBeanServer.queryMBeans(JmxMBeanServer.java:592)
        at app//org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter$JmxListener.unregisterMBean(JmxMetricsReporter.java:555)
        at app//org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter$JmxListener.unregisterAll(JmxMetricsReporter.java:709)
        at app//org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter.close(JmxMetricsReporter.java:759)
        at app//org.apache.solr.metrics.reporters.SolrJmxReporter.close(SolrJmxReporter.java:129)
        at app//org.apache.solr.metrics.SolrMetricManager.lambda$closeReporters$5(SolrMetricManager.java:1157)
        at app//org.apache.solr.metrics.SolrMetricManager$$Lambda$961/0x000000080133f9e8.accept(Unknown Source)
        at java.base@16.0.2/java.lang.Iterable.forEach(Iterable.java:75)
        at app//org.apache.solr.metrics.SolrMetricManager.closeReporters(SolrMetricManager.java:1151)
        at app//org.apache.solr.metrics.SolrCoreMetricManager.close(SolrCoreMetricManager.java:145)
        at app//org.apache.solr.core.SolrCore.doClose(SolrCore.java:1596)
        at app//org.apache.solr.core.SolrCore.close(SolrCore.java:1582)
        at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1100)
        at app//org.apache.solr.core.SolrCore.reload(SolrCore.java:744)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1686)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3134)
        at app//org.apache.solr.core.SolrCore$$Lambda$756/0x0000000801221330.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2499)
        at app//org.apache.solr.cloud.ZkController$$Lambda$951/0x000000080133e028.run(Unknown Source)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   4) Thread[id=11388, name=searcherExecutor-7966-thread-1, state=WAITING, group=TGRP-TestBulkSchemaConcurrent]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   5) Thread[id=11390, name=searcherExecutor-7968-thread-1, state=WAITING, group=TGRP-TestBulkSchemaConcurrent]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   6) Thread[id=11389, name=searcherExecutor-7981-thread-1, state=WAITING, group=TGRP-TestBulkSchemaConcurrent]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 6 threads leaked from SUITE scope at org.apache.solr.schema.TestBulkSchemaConcurrent: 
   1) Thread[id=11255, name=ZKEventListenerThread, state=WAITING, group=TGRP-TestBulkSchemaConcurrent]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:714)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireShared(AbstractQueuedSynchronizer.java:1026)
        at java.base@16.0.2/java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.lock(ReentrantReadWriteLock.java:738)
        at java.management@16.0.2/com.sun.jmx.mbeanserver.Repository.retrieve(Repository.java:484)
        at java.management@16.0.2/com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.getMBean(DefaultMBeanServerInterceptor.java:1077)
        at java.management@16.0.2/com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.getAttribute(DefaultMBeanServerInterceptor.java:637)
        at java.management@16.0.2/com.sun.jmx.mbeanserver.JmxMBeanServer.getAttribute(JmxMBeanServer.java:678)
        at java.management@16.0.2/javax.management.AttributeValueExp.getAttribute(AttributeValueExp.java:162)
        at java.management@16.0.2/javax.management.AttributeValueExp.apply(AttributeValueExp.java:104)
        at java.management@16.0.2/javax.management.BinaryRelQueryExp.apply(BinaryRelQueryExp.java:109)
        at java.management@16.0.2/com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.filterListOfObjectInstances(DefaultMBeanServerInterceptor.java:1599)
        at java.management@16.0.2/com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.queryMBeans(DefaultMBeanServerInterceptor.java:495)
        at java.management@16.0.2/com.sun.jmx.mbeanserver.JmxMBeanServer.queryMBeans(JmxMBeanServer.java:592)
        at app//org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter$JmxListener.unregisterMBean(JmxMetricsReporter.java:555)
        at app//org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter$JmxListener.unregisterAll(JmxMetricsReporter.java:709)
        at app//org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter.close(JmxMetricsReporter.java:759)
        at app//org.apache.solr.metrics.reporters.SolrJmxReporter.close(SolrJmxReporter.java:129)
        at app//org.apache.solr.metrics.SolrMetricManager.lambda$closeReporters$5(SolrMetricManager.java:1157)
        at app//org.apache.solr.metrics.SolrMetricManager$$Lambda$961/0x000000080133f9e8.accept(Unknown Source)
        at java.base@16.0.2/java.lang.Iterable.forEach(Iterable.java:75)
        at app//org.apache.solr.metrics.SolrMetricManager.closeReporters(SolrMetricManager.java:1151)
        at app//org.apache.solr.metrics.SolrCoreMetricManager.close(SolrCoreMetricManager.java:145)
        at app//org.apache.solr.core.SolrCore.doClose(SolrCore.java:1596)
        at app//org.apache.solr.core.SolrCore.close(SolrCore.java:1582)
        at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1100)
        at app//org.apache.solr.core.SolrCore.reload(SolrCore.java:744)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1686)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3134)
        at app//org.apache.solr.core.SolrCore$$Lambda$756/0x0000000801221330.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2499)
        at app//org.apache.solr.cloud.ZkController$$Lambda$951/0x000000080133e028.run(Unknown Source)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   2) Thread[id=11229, name=ZKEventListenerThread, state=WAITING, group=TGRP-TestBulkSchemaConcurrent]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:714)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireShared(AbstractQueuedSynchronizer.java:1026)
        at java.base@16.0.2/java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.lock(ReentrantReadWriteLock.java:738)
        at java.management@16.0.2/com.sun.jmx.mbeanserver.Repository.query(Repository.java:525)
        at java.management@16.0.2/com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.queryMBeansImpl(DefaultMBeanServerInterceptor.java:507)
        at java.management@16.0.2/com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.queryMBeans(DefaultMBeanServerInterceptor.java:476)
        at java.management@16.0.2/com.sun.jmx.mbeanserver.JmxMBeanServer.queryMBeans(JmxMBeanServer.java:592)
        at app//org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter$JmxListener.unregisterMBean(JmxMetricsReporter.java:555)
        at app//org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter$JmxListener.unregisterAll(JmxMetricsReporter.java:709)
        at app//org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter.close(JmxMetricsReporter.java:759)
        at app//org.apache.solr.metrics.reporters.SolrJmxReporter.close(SolrJmxReporter.java:129)
        at app//org.apache.solr.metrics.SolrMetricManager.lambda$closeReporters$5(SolrMetricManager.java:1157)
        at app//org.apache.solr.metrics.SolrMetricManager$$Lambda$961/0x000000080133f9e8.accept(Unknown Source)
        at java.base@16.0.2/java.lang.Iterable.forEach(Iterable.java:75)
        at app//org.apache.solr.metrics.SolrMetricManager.closeReporters(SolrMetricManager.java:1151)
        at app//org.apache.solr.metrics.SolrCoreMetricManager.close(SolrCoreMetricManager.java:145)
        at app//org.apache.solr.core.SolrCore.doClose(SolrCore.java:1596)
        at app//org.apache.solr.core.SolrCore.close(SolrCore.java:1582)
        at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1100)
        at app//org.apache.solr.core.SolrCore.reload(SolrCore.java:744)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1686)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3134)
        at app//org.apache.solr.core.SolrCore$$Lambda$756/0x0000000801221330.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2499)
        at app//org.apache.solr.cloud.ZkController$$Lambda$951/0x000000080133e028.run(Unknown Source)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   3) Thread[id=11252, name=ZKEventListenerThread, state=WAITING, group=TGRP-TestBulkSchemaConcurrent]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:714)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireShared(AbstractQueuedSynchronizer.java:1026)
        at java.base@16.0.2/java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.lock(ReentrantReadWriteLock.java:738)
        at java.management@16.0.2/com.sun.jmx.mbeanserver.Repository.query(Repository.java:525)
        at java.management@16.0.2/com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.queryMBeansImpl(DefaultMBeanServerInterceptor.java:507)
        at java.management@16.0.2/com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.queryMBeans(DefaultMBeanServerInterceptor.java:476)
        at java.management@16.0.2/com.sun.jmx.mbeanserver.JmxMBeanServer.queryMBeans(JmxMBeanServer.java:592)
        at app//org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter$JmxListener.unregisterMBean(JmxMetricsReporter.java:555)
        at app//org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter$JmxListener.unregisterAll(JmxMetricsReporter.java:709)
        at app//org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter.close(JmxMetricsReporter.java:759)
        at app//org.apache.solr.metrics.reporters.SolrJmxReporter.close(SolrJmxReporter.java:129)
        at app//org.apache.solr.metrics.SolrMetricManager.lambda$closeReporters$5(SolrMetricManager.java:1157)
        at app//org.apache.solr.metrics.SolrMetricManager$$Lambda$961/0x000000080133f9e8.accept(Unknown Source)
        at java.base@16.0.2/java.lang.Iterable.forEach(Iterable.java:75)
        at app//org.apache.solr.metrics.SolrMetricManager.closeReporters(SolrMetricManager.java:1151)
        at app//org.apache.solr.metrics.SolrCoreMetricManager.close(SolrCoreMetricManager.java:145)
        at app//org.apache.solr.core.SolrCore.doClose(SolrCore.java:1596)
        at app//org.apache.solr.core.SolrCore.close(SolrCore.java:1582)
        at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1100)
        at app//org.apache.solr.core.SolrCore.reload(SolrCore.java:744)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1686)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3134)
        at app//org.apache.solr.core.SolrCore$$Lambda$756/0x0000000801221330.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2499)
        at app//org.apache.solr.cloud.ZkController$$Lambda$951/0x000000080133e028.run(Unknown Source)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   4) Thread[id=11388, name=searcherExecutor-7966-thread-1, state=WAITING, group=TGRP-TestBulkSchemaConcurrent]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   5) Thread[id=11390, name=searcherExecutor-7968-thread-1, state=WAITING, group=TGRP-TestBulkSchemaConcurrent]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   6) Thread[id=11389, name=searcherExecutor-7981-thread-1, state=WAITING, group=TGRP-TestBulkSchemaConcurrent]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
	at __randomizedtesting.SeedInfo.seed([1A5FAFA6CE7F7FF0]:0)


FAILED:  org.apache.solr.cloud.api.collections.TestHdfsCloudBackupRestore.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.cloud.api.collections.TestHdfsCloudBackupRestore: 
   1) Thread[id=5036, name=indexFetcher-671-thread-1, state=TIMED_WAITING, group=TGRP-TestHdfsCloudBackupRestore]
        at java.base@16.0.2/java.lang.Object.wait(Native Method)
        at app//org.apache.solr.core.CachingDirectoryFactory.close(CachingDirectoryFactory.java:177)
        at app//org.apache.solr.core.SolrCore.doClose(SolrCore.java:1708)
        at app//org.apache.solr.core.SolrCore.close(SolrCore.java:1582)
        at app//org.apache.solr.handler.IndexFetcher.openNewSearcherAndUpdateCommitPoint(IndexFetcher.java:922)
        at app//org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:662)
        at app//org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:360)
        at app//org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:460)
        at app//org.apache.solr.handler.ReplicationHandler.lambda$setupPolling$13(ReplicationHandler.java:1227)
        at app//org.apache.solr.handler.ReplicationHandler$$Lambda$1114/0x00000008018a0b98.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
        at java.base@16.0.2/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.cloud.api.collections.TestHdfsCloudBackupRestore: 
   1) Thread[id=5036, name=indexFetcher-671-thread-1, state=TIMED_WAITING, group=TGRP-TestHdfsCloudBackupRestore]
        at java.base@16.0.2/java.lang.Object.wait(Native Method)
        at app//org.apache.solr.core.CachingDirectoryFactory.close(CachingDirectoryFactory.java:177)
        at app//org.apache.solr.core.SolrCore.doClose(SolrCore.java:1708)
        at app//org.apache.solr.core.SolrCore.close(SolrCore.java:1582)
        at app//org.apache.solr.handler.IndexFetcher.openNewSearcherAndUpdateCommitPoint(IndexFetcher.java:922)
        at app//org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:662)
        at app//org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:360)
        at app//org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:460)
        at app//org.apache.solr.handler.ReplicationHandler.lambda$setupPolling$13(ReplicationHandler.java:1227)
        at app//org.apache.solr.handler.ReplicationHandler$$Lambda$1114/0x00000008018a0b98.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
        at java.base@16.0.2/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305)
        at java.base@16.0.2/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
	at __randomizedtesting.SeedInfo.seed([1A5FAFA6CE7F7FF0]:0)


FAILED:  org.apache.solr.cloud.hdfs.HDFSCollectionsAPITest.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.cloud.hdfs.HDFSCollectionsAPITest: 
   1) Thread[id=4056, name=Command processor, state=WAITING, group=TGRP-HDFSCollectionsAPITest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at app//org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1331)
        at app//org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1315)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.cloud.hdfs.HDFSCollectionsAPITest: 
   1) Thread[id=4056, name=Command processor, state=WAITING, group=TGRP-HDFSCollectionsAPITest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at app//org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1331)
        at app//org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1315)
	at __randomizedtesting.SeedInfo.seed([1A5FAFA6CE7F7FF0]:0)


FAILED:  org.apache.solr.core.backup.repository.HdfsBackupRepositoryIntegrationTest.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.core.backup.repository.HdfsBackupRepositoryIntegrationTest: 
   1) Thread[id=90, name=Command processor, state=WAITING, group=TGRP-HdfsBackupRepositoryIntegrationTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at app//org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1331)
        at app//org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1315)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.core.backup.repository.HdfsBackupRepositoryIntegrationTest: 
   1) Thread[id=90, name=Command processor, state=WAITING, group=TGRP-HdfsBackupRepositoryIntegrationTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at app//org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1331)
        at app//org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1315)
	at __randomizedtesting.SeedInfo.seed([1A5FAFA6CE7F7FF0]:0)


FAILED:  org.apache.solr.store.hdfs.HdfsDirectoryTest.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.store.hdfs.HdfsDirectoryTest: 
   1) Thread[id=1085, name=Command processor, state=WAITING, group=TGRP-HdfsDirectoryTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at app//org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1331)
        at app//org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1315)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.store.hdfs.HdfsDirectoryTest: 
   1) Thread[id=1085, name=Command processor, state=WAITING, group=TGRP-HdfsDirectoryTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at app//org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1331)
        at app//org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1315)
	at __randomizedtesting.SeedInfo.seed([1A5FAFA6CE7F7FF0]:0)


FAILED:  org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=1131, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:156)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:89)
        at app//org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:342)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:82)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getLines(AbstractAnalysisFactory.java:302)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getWordSet(AbstractAnalysisFactory.java:293)
        at app//org.apache.lucene.analysis.en.AbstractWordsFileFilterFactory.inform(AbstractWordsFileFilterFactory.java:88)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:723)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1341)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1203)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1179)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:615)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:177)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:107)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:244)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:314)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:241)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$652/0x00000008011d3d08.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:46)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:239)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1682)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3134)
        at app//org.apache.solr.core.SolrCore$$Lambda$761/0x0000000801297450.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2499)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1202/0x0000000801687af8.run(Unknown Source)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=1131, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:156)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:89)
        at app//org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:342)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:82)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getLines(AbstractAnalysisFactory.java:302)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getWordSet(AbstractAnalysisFactory.java:293)
        at app//org.apache.lucene.analysis.en.AbstractWordsFileFilterFactory.inform(AbstractWordsFileFilterFactory.java:88)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:723)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1341)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1203)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1179)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:615)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:177)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:107)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:244)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:314)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:241)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$652/0x00000008011d3d08.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:46)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:239)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1682)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3134)
        at app//org.apache.solr.core.SolrCore$$Lambda$761/0x0000000801297450.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2499)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1202/0x0000000801687af8.run(Unknown Source)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
	at __randomizedtesting.SeedInfo.seed([1A5FAFA6CE7F7FF0]:0)

[JENKINS] Solr-main-MacOSX (64bit/jdk-17.0.2) - Build # 677 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/677/
Java: 64bit/jdk-17.0.2 -XX:+UseCompressedOops -XX:+UseG1GC

7 tests failed.
FAILED:  org.apache.solr.cloud.HttpPartitionOnCommitTest.test

Error Message:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:64407/_/f: Underlying core creation failed while creating collection: c8n_2x2_commits

Stack Trace:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:64407/_/f: Underlying core creation failed while creating collection: c8n_2x2_commits
	at __randomizedtesting.SeedInfo.seed([AAA08AEBABF4ED39:22F4B531050880C1]:0)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:668)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:239)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:222)
	at app//org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1290)
	at app//org.apache.solr.cloud.AbstractBasicDistributedZkTestBase.createCollection(AbstractBasicDistributedZkTestBase.java:803)
	at app//org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1747)
	at app//org.apache.solr.cloud.HttpPartitionOnCommitTest.multiShardTest(HttpPartitionOnCommitTest.java:76)
	at app//org.apache.solr.cloud.HttpPartitionOnCommitTest.test(HttpPartitionOnCommitTest.java:67)
	at java.base@17.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base@17.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base@17.0.2/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base@17.0.2/java.lang.reflect.Method.invoke(Method.java:568)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1081)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1052)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)


FAILED:  org.apache.solr.cloud.ReindexCollectionTest.testAbort

Error Message:
java.lang.AssertionError: {responseHeader={status=0, QTime=24}, status={state=running, msg=found [b5fb0b7f-e17b-45bd-8494-aaea483a6592] in running tasks}}

Stack Trace:
java.lang.AssertionError: {responseHeader={status=0, QTime=24}, status={state=running, msg=found [b5fb0b7f-e17b-45bd-8494-aaea483a6592] in running tasks}}
	at __randomizedtesting.SeedInfo.seed([AAA08AEBABF4ED39:CCC33D7CDF2E7F40]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.junit.Assert.assertTrue(Assert.java:42)
	at org.junit.Assert.assertNotNull(Assert.java:713)
	at org.apache.solr.cloud.ReindexCollectionTest.testAbort(ReindexCollectionTest.java:377)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:568)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:833)


FAILED:  org.apache.solr.cloud.api.collections.TestHdfsCloudBackupRestore.test

Error Message:
java.lang.AssertionError: expected:<COMPLETED> but was:<RUNNING>

Stack Trace:
java.lang.AssertionError: expected:<COMPLETED> but was:<RUNNING>
	at __randomizedtesting.SeedInfo.seed([AAA08AEBABF4ED39:22F4B531050880C1]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.junit.Assert.failNotEquals(Assert.java:835)
	at org.junit.Assert.assertEquals(Assert.java:120)
	at org.junit.Assert.assertEquals(Assert.java:146)
	at org.apache.solr.cloud.api.collections.AbstractCloudBackupRestoreTestCase.testBackupAndRestore(AbstractCloudBackupRestoreTestCase.java:357)
	at org.apache.solr.cloud.api.collections.AbstractCloudBackupRestoreTestCase.test(AbstractCloudBackupRestoreTestCase.java:157)
	at org.apache.solr.cloud.api.collections.TestHdfsCloudBackupRestore.test(TestHdfsCloudBackupRestore.java:227)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:568)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:833)


FAILED:  org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=1150, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@17.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:156)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:89)
        at app//org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:342)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:82)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getSnowballWordSet(AbstractAnalysisFactory.java:321)
        at app//org.apache.lucene.analysis.en.AbstractWordsFileFilterFactory.inform(AbstractWordsFileFilterFactory.java:90)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:723)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1341)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1194)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1179)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:615)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:177)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:107)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:244)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:314)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:241)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$631/0x0000000801149ca8.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:46)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:239)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1682)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3134)
        at app//org.apache.solr.core.SolrCore$$Lambda$740/0x0000000801209c88.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2499)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1197/0x000000080166c410.run(Unknown Source)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=1150, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@17.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:156)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:89)
        at app//org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:342)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:82)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getSnowballWordSet(AbstractAnalysisFactory.java:321)
        at app//org.apache.lucene.analysis.en.AbstractWordsFileFilterFactory.inform(AbstractWordsFileFilterFactory.java:90)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:723)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1341)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1194)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1179)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:615)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:177)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:107)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:244)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:314)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:241)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$631/0x0000000801149ca8.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:46)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:239)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1682)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3134)
        at app//org.apache.solr.core.SolrCore$$Lambda$740/0x0000000801209c88.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2499)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1197/0x000000080166c410.run(Unknown Source)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
	at __randomizedtesting.SeedInfo.seed([AAA08AEBABF4ED39]:0)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
java.security.AccessControlException: access denied ("java.io.FilePermission" "/Users/jenkins/workspace/Solr-main-MacOSX/solr/core/build/resources/test/solr/filestore" "write")

Stack Trace:
java.security.AccessControlException: access denied ("java.io.FilePermission" "/Users/jenkins/workspace/Solr-main-MacOSX/solr/core/build/resources/test/solr/filestore" "write")
	at __randomizedtesting.SeedInfo.seed([AAA08AEBABF4ED39]:0)
	at java.base/java.security.AccessControlContext.checkPermission(AccessControlContext.java:485)
	at java.base/java.security.AccessController.checkPermission(AccessController.java:1068)
	at java.base/java.lang.SecurityManager.checkPermission(SecurityManager.java:416)
	at java.base/java.lang.SecurityManager.checkWrite(SecurityManager.java:847)
	at java.base/sun.nio.fs.UnixPath.checkWrite(UnixPath.java:787)
	at java.base/sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:389)
	at java.base/java.nio.file.Files.createDirectory(Files.java:700)
	at java.base/java.nio.file.Files.createAndCheckIsDirectory(Files.java:807)
	at java.base/java.nio.file.Files.createDirectories(Files.java:753)
	at org.apache.solr.filestore.DistribPackageStore.getPackageStoreDirPath(DistribPackageStore.java:584)
	at org.apache.solr.filestore.DistribPackageStore.ensurePackageStoreDir(DistribPackageStore.java:567)
	at org.apache.solr.filestore.DistribPackageStore.<init>(DistribPackageStore.java:83)
	at org.apache.solr.filestore.PackageStoreAPI.<init>(PackageStoreAPI.java:76)
	at org.apache.solr.core.CoreContainer.load(CoreContainer.java:721)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:184)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:156)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:162)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:112)
	at org.apache.solr.SolrTestCaseJ4.createCore(SolrTestCaseJ4.java:813)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:803)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:650)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:639)
	at org.apache.solr.cloud.AbstractZkTestCase.azt_beforeClass(AbstractZkTestCase.java:68)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:568)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:882)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:833)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 8 threads leaked from SUITE scope at org.apache.solr.rest.TestManagedResourceStorage: 
   1) Thread[id=24489, name=SUITE-TestManagedResourceStorage-seed#[AAA08AEBABF4ED39]-worker-SendThread(127.0.0.1:56863), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:214)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1380)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanAndNotifyState(ClientCnxn.java:1321)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1299)
   2) Thread[id=24493, name=OverseerCollectionConfigSetProcessor-72097857286438915-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:271)
        at app//org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:258)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   3) Thread[id=24491, name=zkConnectionManagerCallback-15875-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3463)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3434)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at java.base@17.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   4) Thread[id=24490, name=SUITE-TestManagedResourceStorage-seed#[AAA08AEBABF4ED39]-worker-EventThread, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3463)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3434)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at java.base@17.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:551)
   5) Thread[id=24485, name=h2sc-15870-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@17.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$278/0x0000000800f201f8.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$235/0x0000000800ed8c78.run(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   6) Thread[id=24492, name=OverseerStateUpdate-72097857286438915-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:294)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   7) Thread[id=24482, name=httpShardExecutor-15868-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@17.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$278/0x0000000800f201f8.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$235/0x0000000800ed8c78.run(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   8) Thread[id=24494, name=zkCallback-15874-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@17.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 8 threads leaked from SUITE scope at org.apache.solr.rest.TestManagedResourceStorage: 
   1) Thread[id=24489, name=SUITE-TestManagedResourceStorage-seed#[AAA08AEBABF4ED39]-worker-SendThread(127.0.0.1:56863), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:214)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1380)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanAndNotifyState(ClientCnxn.java:1321)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1299)
   2) Thread[id=24493, name=OverseerCollectionConfigSetProcessor-72097857286438915-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:271)
        at app//org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:258)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   3) Thread[id=24491, name=zkConnectionManagerCallback-15875-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3463)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3434)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at java.base@17.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   4) Thread[id=24490, name=SUITE-TestManagedResourceStorage-seed#[AAA08AEBABF4ED39]-worker-EventThread, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3463)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3434)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at java.base@17.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:551)
   5) Thread[id=24485, name=h2sc-15870-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@17.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$278/0x0000000800f201f8.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$235/0x0000000800ed8c78.run(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   6) Thread[id=24492, name=OverseerStateUpdate-72097857286438915-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:294)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   7) Thread[id=24482, name=httpShardExecutor-15868-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@17.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$278/0x0000000800f201f8.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$235/0x0000000800ed8c78.run(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   8) Thread[id=24494, name=zkCallback-15874-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@17.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
	at __randomizedtesting.SeedInfo.seed([AAA08AEBABF4ED39]:0)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=24489, name=SUITE-TestManagedResourceStorage-seed#[AAA08AEBABF4ED39]-worker-SendThread(127.0.0.1:56863), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:362)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1197)
   2) Thread[id=24491, name=zkConnectionManagerCallback-15875-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3463)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3434)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at java.base@17.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   3) Thread[id=24485, name=h2sc-15870-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@17.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$278/0x0000000800f201f8.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$235/0x0000000800ed8c78.run(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   4) Thread[id=24482, name=httpShardExecutor-15868-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@17.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$278/0x0000000800f201f8.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$235/0x0000000800ed8c78.run(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   5) Thread[id=24494, name=zkCallback-15874-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@17.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=24489, name=SUITE-TestManagedResourceStorage-seed#[AAA08AEBABF4ED39]-worker-SendThread(127.0.0.1:56863), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:362)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1197)
   2) Thread[id=24491, name=zkConnectionManagerCallback-15875-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3463)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3434)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at java.base@17.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   3) Thread[id=24485, name=h2sc-15870-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@17.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$278/0x0000000800f201f8.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$235/0x0000000800ed8c78.run(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   4) Thread[id=24482, name=httpShardExecutor-15868-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@17.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$278/0x0000000800f201f8.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$235/0x0000000800ed8c78.run(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   5) Thread[id=24494, name=zkCallback-15874-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@17.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
	at __randomizedtesting.SeedInfo.seed([AAA08AEBABF4ED39]:0)

[JENKINS] Solr-main-MacOSX (64bit/jdk-11.0.13) - Build # 676 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/676/
Java: 64bit/jdk-11.0.13 -XX:+UseCompressedOops -XX:+UseConcMarkSweepGC

6 tests failed.
FAILED:  org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=4558, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:156)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:89)
        at app//org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:342)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:82)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getLines(AbstractAnalysisFactory.java:302)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getWordSet(AbstractAnalysisFactory.java:293)
        at app//org.apache.lucene.analysis.en.AbstractWordsFileFilterFactory.inform(AbstractWordsFileFilterFactory.java:88)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:723)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1341)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1194)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1179)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:615)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:177)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:107)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:244)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:314)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:241)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$599/0x000000080069fc40.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:46)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:239)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1686)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3134)
        at app//org.apache.solr.core.SolrCore$$Lambda$722/0x000000080080d840.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2499)
        at app//org.apache.solr.cloud.ZkController$$Lambda$847/0x00000008008a4840.run(Unknown Source)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=4558, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:156)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:89)
        at app//org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:342)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:82)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getLines(AbstractAnalysisFactory.java:302)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getWordSet(AbstractAnalysisFactory.java:293)
        at app//org.apache.lucene.analysis.en.AbstractWordsFileFilterFactory.inform(AbstractWordsFileFilterFactory.java:88)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:723)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1341)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1194)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1179)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:615)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:177)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:107)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:244)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:314)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:241)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$599/0x000000080069fc40.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:46)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:239)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1686)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3134)
        at app//org.apache.solr.core.SolrCore$$Lambda$722/0x000000080080d840.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2499)
        at app//org.apache.solr.cloud.ZkController$$Lambda$847/0x00000008008a4840.run(Unknown Source)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
	at __randomizedtesting.SeedInfo.seed([24CD51A70119C704]:0)


FAILED:  org.apache.solr.client.solrj.impl.Http2SolrClientCompatibilityTest.testConnectToNewNodesUsingHttp1

Error Message:
java.lang.Exception: Test abandoned because suite timeout was reached.

Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
	at __randomizedtesting.SeedInfo.seed([24CD51A70119C704]:0)


FAILED:  org.apache.solr.client.solrj.impl.Http2SolrClientCompatibilityTest.classMethod

Error Message:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).

Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
	at __randomizedtesting.SeedInfo.seed([24CD51A70119C704]:0)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
java.security.AccessControlException: access denied ("java.io.FilePermission" "/Users/jenkins/workspace/Solr-main-MacOSX/solr/core/build/resources/test/solr/filestore" "write")

Stack Trace:
java.security.AccessControlException: access denied ("java.io.FilePermission" "/Users/jenkins/workspace/Solr-main-MacOSX/solr/core/build/resources/test/solr/filestore" "write")
	at __randomizedtesting.SeedInfo.seed([24CD51A70119C704]:0)
	at java.base/java.security.AccessControlContext.checkPermission(AccessControlContext.java:472)
	at java.base/java.security.AccessController.checkPermission(AccessController.java:897)
	at java.base/java.lang.SecurityManager.checkPermission(SecurityManager.java:322)
	at java.base/java.lang.SecurityManager.checkWrite(SecurityManager.java:752)
	at java.base/sun.nio.fs.UnixPath.checkWrite(UnixPath.java:824)
	at java.base/sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:381)
	at java.base/java.nio.file.Files.createDirectory(Files.java:690)
	at java.base/java.nio.file.Files.createAndCheckIsDirectory(Files.java:797)
	at java.base/java.nio.file.Files.createDirectories(Files.java:743)
	at org.apache.solr.filestore.DistribPackageStore.getPackageStoreDirPath(DistribPackageStore.java:584)
	at org.apache.solr.filestore.DistribPackageStore.ensurePackageStoreDir(DistribPackageStore.java:567)
	at org.apache.solr.filestore.DistribPackageStore.<init>(DistribPackageStore.java:83)
	at org.apache.solr.filestore.PackageStoreAPI.<init>(PackageStoreAPI.java:76)
	at org.apache.solr.core.CoreContainer.load(CoreContainer.java:725)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:184)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:156)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:162)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:112)
	at org.apache.solr.SolrTestCaseJ4.createCore(SolrTestCaseJ4.java:813)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:803)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:650)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:639)
	at org.apache.solr.cloud.AbstractZkTestCase.azt_beforeClass(AbstractZkTestCase.java:68)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:882)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:829)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 8 threads leaked from SUITE scope at org.apache.solr.rest.TestManagedResourceStorage: 
   1) Thread[id=24843, name=OverseerStateUpdate-72096933534171139-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:294)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   2) Thread[id=24836, name=h2sc-16108-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$256/0x000000080040b840.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$219/0x00000008003bf040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   3) Thread[id=24833, name=httpShardExecutor-16106-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$256/0x000000080040b840.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$219/0x00000008003bf040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   4) Thread[id=24842, name=zkConnectionManagerCallback-16113-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   5) Thread[id=24840, name=SUITE-TestManagedResourceStorage-seed#[24CD51A70119C704]-worker-SendThread(127.0.0.1:51036), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:214)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1380)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanAndNotifyState(ClientCnxn.java:1321)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1299)
   6) Thread[id=24841, name=SUITE-TestManagedResourceStorage-seed#[24CD51A70119C704]-worker-EventThread, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:551)
   7) Thread[id=24845, name=zkCallback-16112-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   8) Thread[id=24844, name=OverseerCollectionConfigSetProcessor-72096933534171139-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:271)
        at app//org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:258)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 8 threads leaked from SUITE scope at org.apache.solr.rest.TestManagedResourceStorage: 
   1) Thread[id=24843, name=OverseerStateUpdate-72096933534171139-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:294)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   2) Thread[id=24836, name=h2sc-16108-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$256/0x000000080040b840.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$219/0x00000008003bf040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   3) Thread[id=24833, name=httpShardExecutor-16106-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$256/0x000000080040b840.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$219/0x00000008003bf040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   4) Thread[id=24842, name=zkConnectionManagerCallback-16113-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   5) Thread[id=24840, name=SUITE-TestManagedResourceStorage-seed#[24CD51A70119C704]-worker-SendThread(127.0.0.1:51036), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:214)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1380)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanAndNotifyState(ClientCnxn.java:1321)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1299)
   6) Thread[id=24841, name=SUITE-TestManagedResourceStorage-seed#[24CD51A70119C704]-worker-EventThread, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:551)
   7) Thread[id=24845, name=zkCallback-16112-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   8) Thread[id=24844, name=OverseerCollectionConfigSetProcessor-72096933534171139-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:271)
        at app//org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:258)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
	at __randomizedtesting.SeedInfo.seed([24CD51A70119C704]:0)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=24836, name=h2sc-16108-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$256/0x000000080040b840.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$219/0x00000008003bf040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   2) Thread[id=24833, name=httpShardExecutor-16106-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$256/0x000000080040b840.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$219/0x00000008003bf040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   3) Thread[id=24842, name=zkConnectionManagerCallback-16113-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   4) Thread[id=24840, name=SUITE-TestManagedResourceStorage-seed#[24CD51A70119C704]-worker-SendThread(127.0.0.1:51036), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:362)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1197)
   5) Thread[id=24845, name=zkCallback-16112-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=24836, name=h2sc-16108-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$256/0x000000080040b840.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$219/0x00000008003bf040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   2) Thread[id=24833, name=httpShardExecutor-16106-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$256/0x000000080040b840.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$219/0x00000008003bf040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   3) Thread[id=24842, name=zkConnectionManagerCallback-16113-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   4) Thread[id=24840, name=SUITE-TestManagedResourceStorage-seed#[24CD51A70119C704]-worker-SendThread(127.0.0.1:51036), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:362)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1197)
   5) Thread[id=24845, name=zkCallback-16112-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
	at __randomizedtesting.SeedInfo.seed([24CD51A70119C704]:0)

[JENKINS] Solr-main-MacOSX (64bit/jdk-17.0.2) - Build # 675 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/675/
Java: 64bit/jdk-17.0.2 -XX:-UseCompressedOops -XX:+UseShenandoahGC

9 tests failed.
FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test

Error Message:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:53569/i_/u: Underlying core creation failed while creating collection: c8n_1x3_lf

Stack Trace:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:53569/i_/u: Underlying core creation failed while creating collection: c8n_1x3_lf
	at __randomizedtesting.SeedInfo.seed([EC65E6DD9DF88F81:6431D9073304E279]:0)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:668)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:239)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:222)
	at app//org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1290)
	at app//org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1806)
	at app//org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1827)
	at app//org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1747)
	at app//org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:66)
	at app//org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:58)
	at java.base@17.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base@17.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base@17.0.2/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base@17.0.2/java.lang.reflect.Method.invoke(Method.java:568)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1081)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1052)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)


FAILED:  org.apache.solr.cloud.TestLeaderElectionZkExpiry.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.cloud.TestLeaderElectionZkExpiry: 
   1) Thread[id=23888, name=zkConnectionManagerCallback-14548-thread-1-EventThread, state=BLOCKED, group=TGRP-TestLeaderElectionZkExpiry]
        at app//org.apache.solr.common.cloud.ZkStateReader.createClusterStateWatchersAndUpdate(ZkStateReader.java:404)
        at app//org.apache.solr.cloud.ZkController$1.command(ZkController.java:336)
        at app//org.apache.solr.common.cloud.ConnectionManager$1.update(ConnectionManager.java:170)
        at app//org.apache.solr.common.cloud.DefaultConnectionStrategy.reconnect(DefaultConnectionStrategy.java:57)
        at app//org.apache.solr.common.cloud.ConnectionManager.process(ConnectionManager.java:152)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:580)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:555)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.cloud.TestLeaderElectionZkExpiry: 
   1) Thread[id=23888, name=zkConnectionManagerCallback-14548-thread-1-EventThread, state=BLOCKED, group=TGRP-TestLeaderElectionZkExpiry]
        at app//org.apache.solr.common.cloud.ZkStateReader.createClusterStateWatchersAndUpdate(ZkStateReader.java:404)
        at app//org.apache.solr.cloud.ZkController$1.command(ZkController.java:336)
        at app//org.apache.solr.common.cloud.ConnectionManager$1.update(ConnectionManager.java:170)
        at app//org.apache.solr.common.cloud.DefaultConnectionStrategy.reconnect(DefaultConnectionStrategy.java:57)
        at app//org.apache.solr.common.cloud.ConnectionManager.process(ConnectionManager.java:152)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:580)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:555)
	at __randomizedtesting.SeedInfo.seed([EC65E6DD9DF88F81]:0)


FAILED:  org.apache.solr.cloud.TestRebalanceLeaders.testRebalanceLeaders

Error Message:
org.apache.solr.client.solrj.SolrServerException: Timeout occurred while waiting response from server at: http://127.0.0.1:52899/solr

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occurred while waiting response from server at: http://127.0.0.1:52899/solr
	at __randomizedtesting.SeedInfo.seed([EC65E6DD9DF88F81:F28AD42DA553C3F1]:0)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:679)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:239)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:222)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:389)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:359)
	at app//org.apache.solr.client.solrj.impl.BaseCloudSolrClient.sendRequest(BaseCloudSolrClient.java:1134)
	at app//org.apache.solr.client.solrj.impl.BaseCloudSolrClient.requestWithRetryOnStaleState(BaseCloudSolrClient.java:898)
	at app//org.apache.solr.client.solrj.impl.BaseCloudSolrClient.request(BaseCloudSolrClient.java:832)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:226)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:243)
	at app//org.apache.solr.cloud.TestRebalanceLeaders.rebalanceLeaderUsingSolrJAPI(TestRebalanceLeaders.java:373)
	at app//org.apache.solr.cloud.TestRebalanceLeaders.sendRebalanceCommand(TestRebalanceLeaders.java:279)
	at app//org.apache.solr.cloud.TestRebalanceLeaders.testRebalanceLeaders(TestRebalanceLeaders.java:135)
	at java.base@17.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base@17.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base@17.0.2/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base@17.0.2/java.lang.reflect.Method.invoke(Method.java:568)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.base/sun.nio.ch.NioSocketImpl.timedRead(NioSocketImpl.java:283)
	at java.base/sun.nio.ch.NioSocketImpl.implRead(NioSocketImpl.java:309)
	at java.base/sun.nio.ch.NioSocketImpl.read(NioSocketImpl.java:350)
	at java.base/sun.nio.ch.NioSocketImpl$1.read(NioSocketImpl.java:803)
	at java.base/java.net.Socket$SocketInputStream.read(Socket.java:966)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:280)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:157)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:551)
	... 58 more


FAILED:  org.apache.solr.core.TestLazyCores.testDontEvictUsedCore

Error Message:
java.lang.AssertionError

Stack Trace:
java.lang.AssertionError
	at __randomizedtesting.SeedInfo.seed([EC65E6DD9DF88F81:9BF62704C64DD705]:0)
	at org.junit.Assert.fail(Assert.java:87)
	at org.junit.Assert.assertTrue(Assert.java:42)
	at org.junit.Assert.assertTrue(Assert.java:53)
	at org.apache.solr.core.TestLazyCores.testDontEvictUsedCore(TestLazyCores.java:937)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:568)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:833)


FAILED:  org.apache.solr.handler.TestSolrConfigHandlerConcurrent.test

Error Message:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:54568: Underlying core creation failed while creating collection: control_collection

Stack Trace:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:54568: Underlying core creation failed while creating collection: control_collection
	at __randomizedtesting.SeedInfo.seed([EC65E6DD9DF88F81:6431D9073304E279]:0)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:668)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:239)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:222)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:389)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:359)
	at app//org.apache.solr.client.solrj.impl.BaseCloudSolrClient.sendRequest(BaseCloudSolrClient.java:1134)
	at app//org.apache.solr.client.solrj.impl.BaseCloudSolrClient.requestWithRetryOnStaleState(BaseCloudSolrClient.java:898)
	at app//org.apache.solr.client.solrj.impl.BaseCloudSolrClient.request(BaseCloudSolrClient.java:832)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:226)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:243)
	at app//org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:339)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1079)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1052)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)


FAILED:  org.apache.solr.cloud.HttpPartitionOnCommitTest.test

Error Message:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:61915/_wxk: Underlying core creation failed while creating collection: control_collection

Stack Trace:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:61915/_wxk: Underlying core creation failed while creating collection: control_collection
	at __randomizedtesting.SeedInfo.seed([EC65E6DD9DF88F81:6431D9073304E279]:0)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:668)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:239)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:222)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:389)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:359)
	at app//org.apache.solr.client.solrj.impl.BaseCloudSolrClient.sendRequest(BaseCloudSolrClient.java:1134)
	at app//org.apache.solr.client.solrj.impl.BaseCloudSolrClient.requestWithRetryOnStaleState(BaseCloudSolrClient.java:898)
	at app//org.apache.solr.client.solrj.impl.BaseCloudSolrClient.request(BaseCloudSolrClient.java:832)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:226)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:243)
	at app//org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:339)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1079)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1052)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
java.security.AccessControlException: access denied ("java.io.FilePermission" "/Users/jenkins/workspace/Solr-main-MacOSX/solr/core/build/resources/test/solr/filestore" "write")

Stack Trace:
java.security.AccessControlException: access denied ("java.io.FilePermission" "/Users/jenkins/workspace/Solr-main-MacOSX/solr/core/build/resources/test/solr/filestore" "write")
	at __randomizedtesting.SeedInfo.seed([EC65E6DD9DF88F81]:0)
	at java.base/java.security.AccessControlContext.checkPermission(AccessControlContext.java:485)
	at java.base/java.security.AccessController.checkPermission(AccessController.java:1068)
	at java.base/java.lang.SecurityManager.checkPermission(SecurityManager.java:416)
	at java.base/java.lang.SecurityManager.checkWrite(SecurityManager.java:847)
	at java.base/sun.nio.fs.UnixPath.checkWrite(UnixPath.java:787)
	at java.base/sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:389)
	at java.base/java.nio.file.Files.createDirectory(Files.java:700)
	at java.base/java.nio.file.Files.createAndCheckIsDirectory(Files.java:807)
	at java.base/java.nio.file.Files.createDirectories(Files.java:753)
	at org.apache.solr.filestore.DistribPackageStore.getPackageStoreDirPath(DistribPackageStore.java:584)
	at org.apache.solr.filestore.DistribPackageStore.ensurePackageStoreDir(DistribPackageStore.java:567)
	at org.apache.solr.filestore.DistribPackageStore.<init>(DistribPackageStore.java:83)
	at org.apache.solr.filestore.PackageStoreAPI.<init>(PackageStoreAPI.java:76)
	at org.apache.solr.core.CoreContainer.load(CoreContainer.java:725)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:184)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:156)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:162)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:112)
	at org.apache.solr.SolrTestCaseJ4.createCore(SolrTestCaseJ4.java:813)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:803)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:650)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:639)
	at org.apache.solr.cloud.AbstractZkTestCase.azt_beforeClass(AbstractZkTestCase.java:68)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:568)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:882)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:833)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 8 threads leaked from SUITE scope at org.apache.solr.rest.TestManagedResourceStorage: 
   1) Thread[id=22290, name=SUITE-TestManagedResourceStorage-seed#[EC65E6DD9DF88F81]-worker-SendThread(127.0.0.1:60473), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:214)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1380)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanAndNotifyState(ClientCnxn.java:1321)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1299)
   2) Thread[id=22293, name=OverseerStateUpdate-72096214461448195-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:294)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   3) Thread[id=22292, name=zkConnectionManagerCallback-14713-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3463)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3434)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at java.base@17.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   4) Thread[id=22291, name=SUITE-TestManagedResourceStorage-seed#[EC65E6DD9DF88F81]-worker-EventThread, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3463)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3434)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at java.base@17.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:551)
   5) Thread[id=22294, name=OverseerCollectionConfigSetProcessor-72096214461448195-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:271)
        at app//org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:258)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   6) Thread[id=22286, name=h2sc-14708-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@17.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$282/0x0000000800f5f040.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$242/0x0000000800f0d680.run(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   7) Thread[id=22283, name=httpShardExecutor-14706-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@17.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$282/0x0000000800f5f040.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$242/0x0000000800f0d680.run(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   8) Thread[id=22295, name=zkCallback-14712-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@17.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 8 threads leaked from SUITE scope at org.apache.solr.rest.TestManagedResourceStorage: 
   1) Thread[id=22290, name=SUITE-TestManagedResourceStorage-seed#[EC65E6DD9DF88F81]-worker-SendThread(127.0.0.1:60473), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:214)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1380)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanAndNotifyState(ClientCnxn.java:1321)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1299)
   2) Thread[id=22293, name=OverseerStateUpdate-72096214461448195-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:294)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   3) Thread[id=22292, name=zkConnectionManagerCallback-14713-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3463)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3434)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at java.base@17.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   4) Thread[id=22291, name=SUITE-TestManagedResourceStorage-seed#[EC65E6DD9DF88F81]-worker-EventThread, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3463)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3434)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at java.base@17.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:551)
   5) Thread[id=22294, name=OverseerCollectionConfigSetProcessor-72096214461448195-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:271)
        at app//org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:258)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   6) Thread[id=22286, name=h2sc-14708-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@17.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$282/0x0000000800f5f040.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$242/0x0000000800f0d680.run(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   7) Thread[id=22283, name=httpShardExecutor-14706-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@17.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$282/0x0000000800f5f040.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$242/0x0000000800f0d680.run(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   8) Thread[id=22295, name=zkCallback-14712-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@17.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
	at __randomizedtesting.SeedInfo.seed([EC65E6DD9DF88F81]:0)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=22290, name=SUITE-TestManagedResourceStorage-seed#[EC65E6DD9DF88F81]-worker-SendThread(127.0.0.1:60473), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:362)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1197)
   2) Thread[id=22292, name=zkConnectionManagerCallback-14713-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3463)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3434)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at java.base@17.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   3) Thread[id=22286, name=h2sc-14708-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@17.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$282/0x0000000800f5f040.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$242/0x0000000800f0d680.run(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   4) Thread[id=22283, name=httpShardExecutor-14706-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@17.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$282/0x0000000800f5f040.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$242/0x0000000800f0d680.run(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   5) Thread[id=22295, name=zkCallback-14712-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@17.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=22290, name=SUITE-TestManagedResourceStorage-seed#[EC65E6DD9DF88F81]-worker-SendThread(127.0.0.1:60473), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:362)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1197)
   2) Thread[id=22292, name=zkConnectionManagerCallback-14713-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3463)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3434)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at java.base@17.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   3) Thread[id=22286, name=h2sc-14708-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@17.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$282/0x0000000800f5f040.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$242/0x0000000800f0d680.run(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   4) Thread[id=22283, name=httpShardExecutor-14706-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@17.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$282/0x0000000800f5f040.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$242/0x0000000800f0d680.run(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   5) Thread[id=22295, name=zkCallback-14712-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@17.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
	at __randomizedtesting.SeedInfo.seed([EC65E6DD9DF88F81]:0)

[JENKINS] Solr-main-MacOSX (64bit/jdk-11.0.13) - Build # 674 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/674/
Java: 64bit/jdk-11.0.13 -XX:+UseCompressedOops -XX:+UseParallelGC

6 tests failed.
FAILED:  org.apache.solr.cloud.HttpPartitionOnCommitTest.test

Error Message:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:64534: Underlying core creation failed while creating collection: c8n_1x3_commits

Stack Trace:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:64534: Underlying core creation failed while creating collection: c8n_1x3_commits
	at __randomizedtesting.SeedInfo.seed([1441DB1E985CAC31:9C15E4C436A0C1C9]:0)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:668)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:239)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:222)
	at app//org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1290)
	at app//org.apache.solr.cloud.AbstractBasicDistributedZkTestBase.createCollection(AbstractBasicDistributedZkTestBase.java:803)
	at app//org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1747)
	at app//org.apache.solr.cloud.HttpPartitionOnCommitTest.oneShardTest(HttpPartitionOnCommitTest.java:125)
	at app//org.apache.solr.cloud.HttpPartitionOnCommitTest.test(HttpPartitionOnCommitTest.java:66)
	at java.base@11.0.13/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base@11.0.13/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base@11.0.13/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base@11.0.13/java.lang.reflect.Method.invoke(Method.java:566)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1081)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1052)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)


FAILED:  org.apache.solr.cloud.MultiThreadedOCPTest.test

Error Message:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:57585: Underlying core creation failed while creating collection: control_collection

Stack Trace:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:57585: Underlying core creation failed while creating collection: control_collection
	at __randomizedtesting.SeedInfo.seed([1441DB1E985CAC31:9C15E4C436A0C1C9]:0)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:668)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:239)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:222)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:389)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:359)
	at app//org.apache.solr.client.solrj.impl.BaseCloudSolrClient.sendRequest(BaseCloudSolrClient.java:1134)
	at app//org.apache.solr.client.solrj.impl.BaseCloudSolrClient.requestWithRetryOnStaleState(BaseCloudSolrClient.java:898)
	at app//org.apache.solr.client.solrj.impl.BaseCloudSolrClient.request(BaseCloudSolrClient.java:832)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:226)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:243)
	at app//org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:339)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1079)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1052)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)


FAILED:  org.apache.solr.cloud.SolrCloudExampleTest.testLoadDocsIntoGettingStartedCollection

Error Message:
java.lang.AssertionError: Collection 'gettingstarted' doesn't exist after trying to create it!

Stack Trace:
java.lang.AssertionError: Collection 'gettingstarted' doesn't exist after trying to create it!
	at __randomizedtesting.SeedInfo.seed([1441DB1E985CAC31:722E971A9331597]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.junit.Assert.assertTrue(Assert.java:42)
	at org.apache.solr.cloud.SolrCloudExampleTest.testLoadDocsIntoGettingStartedCollection(SolrCloudExampleTest.java:107)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1081)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1052)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:829)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
java.security.AccessControlException: access denied ("java.io.FilePermission" "/Users/jenkins/workspace/Solr-main-MacOSX/solr/core/build/resources/test/solr/filestore" "write")

Stack Trace:
java.security.AccessControlException: access denied ("java.io.FilePermission" "/Users/jenkins/workspace/Solr-main-MacOSX/solr/core/build/resources/test/solr/filestore" "write")
	at __randomizedtesting.SeedInfo.seed([1441DB1E985CAC31]:0)
	at java.base/java.security.AccessControlContext.checkPermission(AccessControlContext.java:472)
	at java.base/java.security.AccessController.checkPermission(AccessController.java:897)
	at java.base/java.lang.SecurityManager.checkPermission(SecurityManager.java:322)
	at java.base/java.lang.SecurityManager.checkWrite(SecurityManager.java:752)
	at java.base/sun.nio.fs.UnixPath.checkWrite(UnixPath.java:824)
	at java.base/sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:381)
	at java.base/java.nio.file.Files.createDirectory(Files.java:690)
	at java.base/java.nio.file.Files.createAndCheckIsDirectory(Files.java:797)
	at java.base/java.nio.file.Files.createDirectories(Files.java:743)
	at org.apache.solr.filestore.DistribPackageStore.getPackageStoreDirPath(DistribPackageStore.java:584)
	at org.apache.solr.filestore.DistribPackageStore.ensurePackageStoreDir(DistribPackageStore.java:567)
	at org.apache.solr.filestore.DistribPackageStore.<init>(DistribPackageStore.java:83)
	at org.apache.solr.filestore.PackageStoreAPI.<init>(PackageStoreAPI.java:76)
	at org.apache.solr.core.CoreContainer.load(CoreContainer.java:725)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:184)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:156)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:162)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:112)
	at org.apache.solr.SolrTestCaseJ4.createCore(SolrTestCaseJ4.java:813)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:803)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:650)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:639)
	at org.apache.solr.cloud.AbstractZkTestCase.azt_beforeClass(AbstractZkTestCase.java:68)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:882)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:829)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 8 threads leaked from SUITE scope at org.apache.solr.rest.TestManagedResourceStorage: 
   1) Thread[id=26103, name=httpShardExecutor-16078-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$254/0x0000000800414040.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$217/0x00000008003be040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   2) Thread[id=26113, name=OverseerStateUpdate-72095145257140227-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:294)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   3) Thread[id=26115, name=zkCallback-16084-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   4) Thread[id=26110, name=SUITE-TestManagedResourceStorage-seed#[1441DB1E985CAC31]-worker-SendThread(127.0.0.1:55789), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:214)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1380)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanAndNotifyState(ClientCnxn.java:1321)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1299)
   5) Thread[id=26112, name=zkConnectionManagerCallback-16085-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   6) Thread[id=26114, name=OverseerCollectionConfigSetProcessor-72095145257140227-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:271)
        at app//org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:258)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   7) Thread[id=26111, name=SUITE-TestManagedResourceStorage-seed#[1441DB1E985CAC31]-worker-EventThread, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:551)
   8) Thread[id=26106, name=h2sc-16080-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$254/0x0000000800414040.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$217/0x00000008003be040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 8 threads leaked from SUITE scope at org.apache.solr.rest.TestManagedResourceStorage: 
   1) Thread[id=26103, name=httpShardExecutor-16078-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$254/0x0000000800414040.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$217/0x00000008003be040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   2) Thread[id=26113, name=OverseerStateUpdate-72095145257140227-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:294)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   3) Thread[id=26115, name=zkCallback-16084-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   4) Thread[id=26110, name=SUITE-TestManagedResourceStorage-seed#[1441DB1E985CAC31]-worker-SendThread(127.0.0.1:55789), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:214)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1380)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanAndNotifyState(ClientCnxn.java:1321)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1299)
   5) Thread[id=26112, name=zkConnectionManagerCallback-16085-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   6) Thread[id=26114, name=OverseerCollectionConfigSetProcessor-72095145257140227-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:271)
        at app//org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:258)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   7) Thread[id=26111, name=SUITE-TestManagedResourceStorage-seed#[1441DB1E985CAC31]-worker-EventThread, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:551)
   8) Thread[id=26106, name=h2sc-16080-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$254/0x0000000800414040.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$217/0x00000008003be040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
	at __randomizedtesting.SeedInfo.seed([1441DB1E985CAC31]:0)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=26103, name=httpShardExecutor-16078-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$254/0x0000000800414040.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$217/0x00000008003be040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   2) Thread[id=26115, name=zkCallback-16084-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   3) Thread[id=26110, name=SUITE-TestManagedResourceStorage-seed#[1441DB1E985CAC31]-worker-SendThread(127.0.0.1:55789), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:362)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1197)
   4) Thread[id=26112, name=zkConnectionManagerCallback-16085-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   5) Thread[id=26106, name=h2sc-16080-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$254/0x0000000800414040.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$217/0x00000008003be040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=26103, name=httpShardExecutor-16078-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$254/0x0000000800414040.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$217/0x00000008003be040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   2) Thread[id=26115, name=zkCallback-16084-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   3) Thread[id=26110, name=SUITE-TestManagedResourceStorage-seed#[1441DB1E985CAC31]-worker-SendThread(127.0.0.1:55789), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:362)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1197)
   4) Thread[id=26112, name=zkConnectionManagerCallback-16085-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   5) Thread[id=26106, name=h2sc-16080-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$254/0x0000000800414040.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$217/0x00000008003be040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
	at __randomizedtesting.SeedInfo.seed([1441DB1E985CAC31]:0)

[JENKINS] Solr-main-MacOSX (64bit/jdk-16.0.2) - Build # 673 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/673/
Java: 64bit/jdk-16.0.2 -XX:-UseCompressedOops -XX:+UseG1GC

5 tests failed.
FAILED:  org.apache.solr.metrics.reporters.SolrJmxReporterTest.testClosedCore

Error Message:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=4550, name=Thread-1000, state=RUNNABLE, group=TGRP-SolrJmxReporterTest]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=4550, name=Thread-1000, state=RUNNABLE, group=TGRP-SolrJmxReporterTest]
	at __randomizedtesting.SeedInfo.seed([2C7BD65B1EC010:7751ADF67B87A2E0]:0)
Caused by: java.lang.AssertionError
	at __randomizedtesting.SeedInfo.seed([2C7BD65B1EC010]:0)
	at org.junit.Assert.fail(Assert.java:87)
	at org.junit.Assert.assertTrue(Assert.java:42)
	at org.junit.Assert.assertNotNull(Assert.java:713)
	at org.junit.Assert.assertNotNull(Assert.java:723)
	at org.apache.solr.metrics.reporters.SolrJmxReporterTest$1.run(SolrJmxReporterTest.java:213)


FAILED:  org.apache.solr.update.processor.DocExpirationUpdateProcessorFactoryTest.testAutomaticDeletes

Error Message:
java.lang.NullPointerException: Cannot invoke "Object.getClass()" because "tmp" is null

Stack Trace:
java.lang.NullPointerException: Cannot invoke "Object.getClass()" because "tmp" is null
	at __randomizedtesting.SeedInfo.seed([2C7BD65B1EC010:CD3F01ACE668D3EC]:0)
	at org.apache.solr.update.processor.DocExpirationUpdateProcessorFactoryTest.testAutomaticDeletes(DocExpirationUpdateProcessorFactoryTest.java:206)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:78)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:567)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:831)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
java.security.AccessControlException: access denied ("java.io.FilePermission" "/Users/jenkins/workspace/Solr-main-MacOSX/solr/core/build/resources/test/solr/filestore" "write")

Stack Trace:
java.security.AccessControlException: access denied ("java.io.FilePermission" "/Users/jenkins/workspace/Solr-main-MacOSX/solr/core/build/resources/test/solr/filestore" "write")
	at __randomizedtesting.SeedInfo.seed([2C7BD65B1EC010]:0)
	at java.base/java.security.AccessControlContext.checkPermission(AccessControlContext.java:472)
	at java.base/java.security.AccessController.checkPermission(AccessController.java:1036)
	at java.base/java.lang.SecurityManager.checkPermission(SecurityManager.java:408)
	at java.base/java.lang.SecurityManager.checkWrite(SecurityManager.java:838)
	at java.base/sun.nio.fs.UnixPath.checkWrite(UnixPath.java:816)
	at java.base/sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:388)
	at java.base/java.nio.file.Files.createDirectory(Files.java:694)
	at java.base/java.nio.file.Files.createAndCheckIsDirectory(Files.java:801)
	at java.base/java.nio.file.Files.createDirectories(Files.java:747)
	at org.apache.solr.filestore.DistribPackageStore.getPackageStoreDirPath(DistribPackageStore.java:584)
	at org.apache.solr.filestore.DistribPackageStore.ensurePackageStoreDir(DistribPackageStore.java:567)
	at org.apache.solr.filestore.DistribPackageStore.<init>(DistribPackageStore.java:83)
	at org.apache.solr.filestore.PackageStoreAPI.<init>(PackageStoreAPI.java:76)
	at org.apache.solr.core.CoreContainer.load(CoreContainer.java:725)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:184)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:156)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:162)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:112)
	at org.apache.solr.SolrTestCaseJ4.createCore(SolrTestCaseJ4.java:813)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:803)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:650)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:639)
	at org.apache.solr.cloud.AbstractZkTestCase.azt_beforeClass(AbstractZkTestCase.java:68)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:78)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:567)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:882)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:831)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 8 threads leaked from SUITE scope at org.apache.solr.rest.TestManagedResourceStorage: 
   1) Thread[id=24911, name=zkConnectionManagerCallback-16611-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   2) Thread[id=24909, name=SUITE-TestManagedResourceStorage-seed#[2C7BD65B1EC010]-worker-SendThread(127.0.0.1:53689), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:214)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1380)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanAndNotifyState(ClientCnxn.java:1321)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1299)
   3) Thread[id=24912, name=OverseerStateUpdate-72094404950949891-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:294)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   4) Thread[id=24914, name=zkCallback-16610-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   5) Thread[id=24905, name=h2sc-16606-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$271/0x0000000800f219b8.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$234/0x0000000800ed58e8.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   6) Thread[id=24902, name=httpShardExecutor-16604-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$271/0x0000000800f219b8.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$234/0x0000000800ed58e8.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   7) Thread[id=24910, name=SUITE-TestManagedResourceStorage-seed#[2C7BD65B1EC010]-worker-EventThread, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:551)
   8) Thread[id=24913, name=OverseerCollectionConfigSetProcessor-72094404950949891-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:271)
        at app//org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:258)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 8 threads leaked from SUITE scope at org.apache.solr.rest.TestManagedResourceStorage: 
   1) Thread[id=24911, name=zkConnectionManagerCallback-16611-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   2) Thread[id=24909, name=SUITE-TestManagedResourceStorage-seed#[2C7BD65B1EC010]-worker-SendThread(127.0.0.1:53689), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:214)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1380)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanAndNotifyState(ClientCnxn.java:1321)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1299)
   3) Thread[id=24912, name=OverseerStateUpdate-72094404950949891-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:294)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   4) Thread[id=24914, name=zkCallback-16610-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   5) Thread[id=24905, name=h2sc-16606-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$271/0x0000000800f219b8.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$234/0x0000000800ed58e8.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   6) Thread[id=24902, name=httpShardExecutor-16604-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$271/0x0000000800f219b8.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$234/0x0000000800ed58e8.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   7) Thread[id=24910, name=SUITE-TestManagedResourceStorage-seed#[2C7BD65B1EC010]-worker-EventThread, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:551)
   8) Thread[id=24913, name=OverseerCollectionConfigSetProcessor-72094404950949891-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:271)
        at app//org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:258)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
	at __randomizedtesting.SeedInfo.seed([2C7BD65B1EC010]:0)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=24911, name=zkConnectionManagerCallback-16611-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   2) Thread[id=24909, name=SUITE-TestManagedResourceStorage-seed#[2C7BD65B1EC010]-worker-SendThread(127.0.0.1:53689), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:362)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1197)
   3) Thread[id=24914, name=zkCallback-16610-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   4) Thread[id=24905, name=h2sc-16606-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$271/0x0000000800f219b8.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$234/0x0000000800ed58e8.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   5) Thread[id=24902, name=httpShardExecutor-16604-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$271/0x0000000800f219b8.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$234/0x0000000800ed58e8.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=24911, name=zkConnectionManagerCallback-16611-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   2) Thread[id=24909, name=SUITE-TestManagedResourceStorage-seed#[2C7BD65B1EC010]-worker-SendThread(127.0.0.1:53689), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:362)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1197)
   3) Thread[id=24914, name=zkCallback-16610-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   4) Thread[id=24905, name=h2sc-16606-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$271/0x0000000800f219b8.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$234/0x0000000800ed58e8.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   5) Thread[id=24902, name=httpShardExecutor-16604-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$271/0x0000000800f219b8.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$234/0x0000000800ed58e8.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
	at __randomizedtesting.SeedInfo.seed([2C7BD65B1EC010]:0)

[JENKINS] Solr-main-MacOSX (64bit/jdk-11.0.13) - Build # 672 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/672/
Java: 64bit/jdk-11.0.13 -XX:-UseCompressedOops -XX:+UseConcMarkSweepGC

7 tests failed.
FAILED:  org.apache.solr.cloud.HttpPartitionOnCommitTest.test

Error Message:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:49306: Underlying core creation failed while creating collection: c8n_2x2_commits

Stack Trace:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:49306: Underlying core creation failed while creating collection: c8n_2x2_commits
	at __randomizedtesting.SeedInfo.seed([850270D93C5B6265:D564F0392A70F9D]:0)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:668)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:239)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:222)
	at app//org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1290)
	at app//org.apache.solr.cloud.AbstractBasicDistributedZkTestBase.createCollection(AbstractBasicDistributedZkTestBase.java:803)
	at app//org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1747)
	at app//org.apache.solr.cloud.HttpPartitionOnCommitTest.multiShardTest(HttpPartitionOnCommitTest.java:76)
	at app//org.apache.solr.cloud.HttpPartitionOnCommitTest.test(HttpPartitionOnCommitTest.java:67)
	at java.base@11.0.13/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base@11.0.13/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base@11.0.13/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base@11.0.13/java.lang.reflect.Method.invoke(Method.java:566)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1081)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1052)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)


FAILED:  org.apache.solr.core.TestLazyCores.testDontEvictUsedCore

Error Message:
java.lang.AssertionError

Stack Trace:
java.lang.AssertionError
	at __randomizedtesting.SeedInfo.seed([850270D93C5B6265:F291B10067EE3AE1]:0)
	at org.junit.Assert.fail(Assert.java:87)
	at org.junit.Assert.assertTrue(Assert.java:42)
	at org.junit.Assert.assertTrue(Assert.java:53)
	at org.apache.solr.core.TestLazyCores.testDontEvictUsedCore(TestLazyCores.java:937)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:829)


FAILED:  org.apache.solr.handler.TestSolrConfigHandlerConcurrent.test

Error Message:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:49937: Underlying core creation failed while creating collection: control_collection

Stack Trace:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:49937: Underlying core creation failed while creating collection: control_collection
	at __randomizedtesting.SeedInfo.seed([850270D93C5B6265:D564F0392A70F9D]:0)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:668)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:239)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:222)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:389)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:359)
	at app//org.apache.solr.client.solrj.impl.BaseCloudSolrClient.sendRequest(BaseCloudSolrClient.java:1134)
	at app//org.apache.solr.client.solrj.impl.BaseCloudSolrClient.requestWithRetryOnStaleState(BaseCloudSolrClient.java:898)
	at app//org.apache.solr.client.solrj.impl.BaseCloudSolrClient.request(BaseCloudSolrClient.java:832)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:226)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:243)
	at app//org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:339)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1079)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1052)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)


FAILED:  org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=1393, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:156)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:89)
        at app//org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:306)
        at app//org.apache.solr.cloud.ZkController.pathExists(ZkController.java:1139)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:80)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getLines(AbstractAnalysisFactory.java:302)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getWordSet(AbstractAnalysisFactory.java:293)
        at app//org.apache.lucene.analysis.en.AbstractWordsFileFilterFactory.inform(AbstractWordsFileFilterFactory.java:88)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:723)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1341)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1194)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1179)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:615)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:177)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:107)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:244)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:314)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:241)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$586/0x000000014319d108.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:46)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:239)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1686)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3134)
        at app//org.apache.solr.core.SolrCore$$Lambda$709/0x0000000143b5d5b8.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2499)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1127/0x0000000144cb64b0.run(Unknown Source)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=1393, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:156)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:89)
        at app//org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:306)
        at app//org.apache.solr.cloud.ZkController.pathExists(ZkController.java:1139)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:80)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getLines(AbstractAnalysisFactory.java:302)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getWordSet(AbstractAnalysisFactory.java:293)
        at app//org.apache.lucene.analysis.en.AbstractWordsFileFilterFactory.inform(AbstractWordsFileFilterFactory.java:88)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:723)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1341)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1194)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1179)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:615)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:177)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:107)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:244)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:314)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:241)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$586/0x000000014319d108.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:46)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:239)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1686)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3134)
        at app//org.apache.solr.core.SolrCore$$Lambda$709/0x0000000143b5d5b8.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2499)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1127/0x0000000144cb64b0.run(Unknown Source)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
	at __randomizedtesting.SeedInfo.seed([850270D93C5B6265]:0)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
java.security.AccessControlException: access denied ("java.io.FilePermission" "/Users/jenkins/workspace/Solr-main-MacOSX/solr/core/build/resources/test/solr/filestore" "write")

Stack Trace:
java.security.AccessControlException: access denied ("java.io.FilePermission" "/Users/jenkins/workspace/Solr-main-MacOSX/solr/core/build/resources/test/solr/filestore" "write")
	at __randomizedtesting.SeedInfo.seed([850270D93C5B6265]:0)
	at java.base/java.security.AccessControlContext.checkPermission(AccessControlContext.java:472)
	at java.base/java.security.AccessController.checkPermission(AccessController.java:897)
	at java.base/java.lang.SecurityManager.checkPermission(SecurityManager.java:322)
	at java.base/java.lang.SecurityManager.checkWrite(SecurityManager.java:752)
	at java.base/sun.nio.fs.UnixPath.checkWrite(UnixPath.java:824)
	at java.base/sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:381)
	at java.base/java.nio.file.Files.createDirectory(Files.java:690)
	at java.base/java.nio.file.Files.createAndCheckIsDirectory(Files.java:797)
	at java.base/java.nio.file.Files.createDirectories(Files.java:743)
	at org.apache.solr.filestore.DistribPackageStore.getPackageStoreDirPath(DistribPackageStore.java:584)
	at org.apache.solr.filestore.DistribPackageStore.ensurePackageStoreDir(DistribPackageStore.java:567)
	at org.apache.solr.filestore.DistribPackageStore.<init>(DistribPackageStore.java:83)
	at org.apache.solr.filestore.PackageStoreAPI.<init>(PackageStoreAPI.java:76)
	at org.apache.solr.core.CoreContainer.load(CoreContainer.java:725)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:184)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:156)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:162)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:112)
	at org.apache.solr.SolrTestCaseJ4.createCore(SolrTestCaseJ4.java:813)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:803)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:650)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:639)
	at org.apache.solr.cloud.AbstractZkTestCase.azt_beforeClass(AbstractZkTestCase.java:68)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:882)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:829)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 8 threads leaked from SUITE scope at org.apache.solr.rest.TestManagedResourceStorage: 
   1) Thread[id=25055, name=zkCallback-17501-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   2) Thread[id=25053, name=OverseerStateUpdate-72092869568430083-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:294)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   3) Thread[id=25051, name=SUITE-TestManagedResourceStorage-seed#[850270D93C5B6265]-worker-EventThread, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:551)
   4) Thread[id=25046, name=h2sc-17497-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$258/0x000000014172d858.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$221/0x00000001419595b8.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   5) Thread[id=25054, name=OverseerCollectionConfigSetProcessor-72092869568430083-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:271)
        at app//org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:258)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   6) Thread[id=25050, name=SUITE-TestManagedResourceStorage-seed#[850270D93C5B6265]-worker-SendThread(127.0.0.1:54282), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:214)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1380)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanAndNotifyState(ClientCnxn.java:1321)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1299)
   7) Thread[id=25043, name=httpShardExecutor-17495-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$258/0x000000014172d858.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$221/0x00000001419595b8.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   8) Thread[id=25052, name=zkConnectionManagerCallback-17502-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 8 threads leaked from SUITE scope at org.apache.solr.rest.TestManagedResourceStorage: 
   1) Thread[id=25055, name=zkCallback-17501-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   2) Thread[id=25053, name=OverseerStateUpdate-72092869568430083-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:294)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   3) Thread[id=25051, name=SUITE-TestManagedResourceStorage-seed#[850270D93C5B6265]-worker-EventThread, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:551)
   4) Thread[id=25046, name=h2sc-17497-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$258/0x000000014172d858.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$221/0x00000001419595b8.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   5) Thread[id=25054, name=OverseerCollectionConfigSetProcessor-72092869568430083-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:271)
        at app//org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:258)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   6) Thread[id=25050, name=SUITE-TestManagedResourceStorage-seed#[850270D93C5B6265]-worker-SendThread(127.0.0.1:54282), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:214)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1380)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanAndNotifyState(ClientCnxn.java:1321)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1299)
   7) Thread[id=25043, name=httpShardExecutor-17495-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$258/0x000000014172d858.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$221/0x00000001419595b8.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   8) Thread[id=25052, name=zkConnectionManagerCallback-17502-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
	at __randomizedtesting.SeedInfo.seed([850270D93C5B6265]:0)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=25055, name=zkCallback-17501-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   2) Thread[id=25046, name=h2sc-17497-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$258/0x000000014172d858.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$221/0x00000001419595b8.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   3) Thread[id=25050, name=SUITE-TestManagedResourceStorage-seed#[850270D93C5B6265]-worker-SendThread(127.0.0.1:54282), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:362)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1197)
   4) Thread[id=25043, name=httpShardExecutor-17495-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$258/0x000000014172d858.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$221/0x00000001419595b8.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   5) Thread[id=25052, name=zkConnectionManagerCallback-17502-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=25055, name=zkCallback-17501-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   2) Thread[id=25046, name=h2sc-17497-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$258/0x000000014172d858.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$221/0x00000001419595b8.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   3) Thread[id=25050, name=SUITE-TestManagedResourceStorage-seed#[850270D93C5B6265]-worker-SendThread(127.0.0.1:54282), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:362)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1197)
   4) Thread[id=25043, name=httpShardExecutor-17495-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$258/0x000000014172d858.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$221/0x00000001419595b8.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   5) Thread[id=25052, name=zkConnectionManagerCallback-17502-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
	at __randomizedtesting.SeedInfo.seed([850270D93C5B6265]:0)

[JENKINS] Solr-main-MacOSX (64bit/jdk-11.0.13) - Build # 671 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/671/
Java: 64bit/jdk-11.0.13 -XX:+UseCompressedOops -XX:+UseSerialGC

9 tests failed.
FAILED:  org.apache.solr.cloud.OverseerTest.testReplay

Error Message:
java.util.concurrent.TimeoutException: Timeout waiting to see state for collection=collection1 :DocCollection(collection1//collections/collection1/state.json/2 )={
  "pullReplicas":"0",
  "replicationFactor":"1",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node1":{
          "core":"core1",
          "core_node_name":"core_node1",
          "roles":"",
          "node_name":"127.0.0.1:8983_solr",
          "base_url":"http://127.0.0.1:8983/solr",
          "state":"recovering",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT"},
        "core_node2":{
          "core":"core2",
          "core_node_name":"core_node2",
          "roles":"",
          "node_name":"node1:8983_",
          "base_url":"http://node1:8983",
          "state":"recovering",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"1",
  "tlogReplicas":"0",
  "autoCreated":"true"}

Stack Trace:
java.util.concurrent.TimeoutException: Timeout waiting to see state for collection=collection1 :DocCollection(collection1//collections/collection1/state.json/2 )={
  "pullReplicas":"0",
  "replicationFactor":"1",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node1":{
          "core":"core1",
          "core_node_name":"core_node1",
          "roles":"",
          "node_name":"127.0.0.1:8983_solr",
          "base_url":"http://127.0.0.1:8983/solr",
          "state":"recovering",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT"},
        "core_node2":{
          "core":"core2",
          "core_node_name":"core_node2",
          "roles":"",
          "node_name":"node1:8983_",
          "base_url":"http://node1:8983",
          "state":"recovering",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"1",
  "tlogReplicas":"0",
  "autoCreated":"true"}
	at __randomizedtesting.SeedInfo.seed([6C16A6295FEF3291:24DFAE6DB1DC43EA]:0)
	at org.apache.solr.common.cloud.ZkStateReader.waitForState(ZkStateReader.java:1655)
	at org.apache.solr.cloud.OverseerTest.testReplay(OverseerTest.java:1266)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:829)


FAILED:  org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=3459, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:156)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:89)
        at app//org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:306)
        at app//org.apache.solr.cloud.ZkController.pathExists(ZkController.java:1139)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:80)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getLines(AbstractAnalysisFactory.java:302)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getWordSet(AbstractAnalysisFactory.java:293)
        at app//org.apache.lucene.analysis.en.AbstractWordsFileFilterFactory.inform(AbstractWordsFileFilterFactory.java:88)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:723)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1341)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1194)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1179)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:615)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:177)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:107)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:244)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:314)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:241)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$610/0x000000080072b440.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:46)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:239)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1686)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3134)
        at app//org.apache.solr.core.SolrCore$$Lambda$733/0x000000080088d840.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2499)
        at app//org.apache.solr.cloud.ZkController$$Lambda$852/0x000000080092b840.run(Unknown Source)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=3459, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:156)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:89)
        at app//org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:306)
        at app//org.apache.solr.cloud.ZkController.pathExists(ZkController.java:1139)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:80)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getLines(AbstractAnalysisFactory.java:302)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getWordSet(AbstractAnalysisFactory.java:293)
        at app//org.apache.lucene.analysis.en.AbstractWordsFileFilterFactory.inform(AbstractWordsFileFilterFactory.java:88)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:723)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1341)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1194)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1179)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:615)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:177)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:107)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:244)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:314)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:241)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$610/0x000000080072b440.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:46)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:239)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1686)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3134)
        at app//org.apache.solr.core.SolrCore$$Lambda$733/0x000000080088d840.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2499)
        at app//org.apache.solr.cloud.ZkController$$Lambda$852/0x000000080092b840.run(Unknown Source)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
	at __randomizedtesting.SeedInfo.seed([6C16A6295FEF3291]:0)


FAILED:  org.apache.solr.schema.TestManagedSchemaAPI.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=16730, name=qtp1696970714-16730, state=RUNNABLE, group=TGRP-TestManagedSchemaAPI], registration stack trace below.

Stack Trace:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=16730, name=qtp1696970714-16730, state=RUNNABLE, group=TGRP-TestManagedSchemaAPI], registration stack trace below.
	at java.base@11.0.13/java.lang.Thread.getStackTrace(Thread.java:1602)
	at app//com.carrotsearch.randomizedtesting.RandomizedContext.closeAtEnd(RandomizedContext.java:157)
	at app//org.apache.lucene.util.LuceneTestCase.closeAfterSuite(LuceneTestCase.java:798)
	at app//org.apache.lucene.util.LuceneTestCase.wrapDirectory(LuceneTestCase.java:1504)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1368)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1350)
	at app//org.apache.solr.core.MockDirectoryFactory.create(MockDirectoryFactory.java:49)
	at app//org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:360)
	at app//org.apache.solr.core.SolrCore.initSnapshotMetaDataManager(SolrCore.java:582)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1026)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:961)
	at app//org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1423)
	at app//org.apache.solr.core.CoreContainer.create(CoreContainer.java:1297)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:345)
	at app//org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:423)
	at app//org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:197)
	at app//org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:214)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:866)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:830)
	at app//org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:542)
	at app//org.apache.solr.servlet.SolrDispatchFilter.dispatch(SolrDispatchFilter.java:218)
	at app//org.apache.solr.servlet.SolrDispatchFilter.lambda$doFilter$0(SolrDispatchFilter.java:189)
	at app//org.apache.solr.servlet.ServletUtils.traceHttpRequestExecution2(ServletUtils.java:232)
	at app//org.apache.solr.servlet.ServletUtils.rateLimitRequest(ServletUtils.java:208)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:187)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:171)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:189)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1434)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
	at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)
	at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1349)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
	at app//org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:713)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.server.Server.handle(Server.java:516)
	at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:400)
	at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:645)
	at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:392)
	at app//org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:352)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
	at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
Caused by: java.lang.AssertionError: Directory not closed: MockDirectoryWrapper(ByteBuffersDirectory@2fb02f5f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1907246f)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.lucene.util.CloseableDirectory.close(CloseableDirectory.java:43)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:788)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:785)
	at com.carrotsearch.randomizedtesting.RandomizedContext.closeResources(RandomizedContext.java:225)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.afterAlways(RandomizedRunner.java:801)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	... 1 more


FAILED:  org.apache.solr.schema.TestManagedSchemaAPI.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=16730, name=qtp1696970714-16730, state=RUNNABLE, group=TGRP-TestManagedSchemaAPI], registration stack trace below.

Stack Trace:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=16730, name=qtp1696970714-16730, state=RUNNABLE, group=TGRP-TestManagedSchemaAPI], registration stack trace below.
	at java.base@11.0.13/java.lang.Thread.getStackTrace(Thread.java:1602)
	at app//com.carrotsearch.randomizedtesting.RandomizedContext.closeAtEnd(RandomizedContext.java:157)
	at app//org.apache.lucene.util.LuceneTestCase.closeAfterSuite(LuceneTestCase.java:798)
	at app//org.apache.lucene.util.LuceneTestCase.wrapDirectory(LuceneTestCase.java:1504)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1368)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1350)
	at app//org.apache.solr.core.MockDirectoryFactory.create(MockDirectoryFactory.java:49)
	at app//org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:360)
	at app//org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:404)
	at app//org.apache.solr.core.SolrCore.initIndex(SolrCore.java:817)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1030)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:961)
	at app//org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1423)
	at app//org.apache.solr.core.CoreContainer.create(CoreContainer.java:1297)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:345)
	at app//org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:423)
	at app//org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:197)
	at app//org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:214)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:866)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:830)
	at app//org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:542)
	at app//org.apache.solr.servlet.SolrDispatchFilter.dispatch(SolrDispatchFilter.java:218)
	at app//org.apache.solr.servlet.SolrDispatchFilter.lambda$doFilter$0(SolrDispatchFilter.java:189)
	at app//org.apache.solr.servlet.ServletUtils.traceHttpRequestExecution2(ServletUtils.java:232)
	at app//org.apache.solr.servlet.ServletUtils.rateLimitRequest(ServletUtils.java:208)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:187)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:171)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:189)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1434)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
	at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)
	at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1349)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
	at app//org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:713)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.server.Server.handle(Server.java:516)
	at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:400)
	at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:645)
	at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:392)
	at app//org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:352)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
	at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
Caused by: java.lang.AssertionError: Directory not closed: MockDirectoryWrapper(ByteBuffersDirectory@104b0309 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@41adbebe)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.lucene.util.CloseableDirectory.close(CloseableDirectory.java:43)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:788)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:785)
	at com.carrotsearch.randomizedtesting.RandomizedContext.closeResources(RandomizedContext.java:225)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.afterAlways(RandomizedRunner.java:801)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	... 1 more


FAILED:  org.apache.solr.schema.TestManagedSchemaAPI.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=16730, name=qtp1696970714-16730, state=RUNNABLE, group=TGRP-TestManagedSchemaAPI], registration stack trace below.

Stack Trace:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=16730, name=qtp1696970714-16730, state=RUNNABLE, group=TGRP-TestManagedSchemaAPI], registration stack trace below.
	at java.base@11.0.13/java.lang.Thread.getStackTrace(Thread.java:1602)
	at app//com.carrotsearch.randomizedtesting.RandomizedContext.closeAtEnd(RandomizedContext.java:157)
	at app//org.apache.lucene.util.LuceneTestCase.closeAfterSuite(LuceneTestCase.java:798)
	at app//org.apache.lucene.util.LuceneTestCase.wrapDirectory(LuceneTestCase.java:1504)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1368)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1350)
	at app//org.apache.solr.core.MockDirectoryFactory.create(MockDirectoryFactory.java:49)
	at app//org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:360)
	at app//org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:98)
	at app//org.apache.solr.core.SolrCore.initIndex(SolrCore.java:849)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1030)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:961)
	at app//org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1423)
	at app//org.apache.solr.core.CoreContainer.create(CoreContainer.java:1297)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:345)
	at app//org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:423)
	at app//org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:197)
	at app//org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:214)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:866)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:830)
	at app//org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:542)
	at app//org.apache.solr.servlet.SolrDispatchFilter.dispatch(SolrDispatchFilter.java:218)
	at app//org.apache.solr.servlet.SolrDispatchFilter.lambda$doFilter$0(SolrDispatchFilter.java:189)
	at app//org.apache.solr.servlet.ServletUtils.traceHttpRequestExecution2(ServletUtils.java:232)
	at app//org.apache.solr.servlet.ServletUtils.rateLimitRequest(ServletUtils.java:208)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:187)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:171)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:189)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1434)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
	at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)
	at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1349)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
	at app//org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:713)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.server.Server.handle(Server.java:516)
	at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:400)
	at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:645)
	at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:392)
	at app//org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:352)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
	at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
Caused by: java.lang.AssertionError: Directory not closed: MockDirectoryWrapper(ByteBuffersDirectory@2ae36f2b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@217c979c)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.lucene.util.CloseableDirectory.close(CloseableDirectory.java:43)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:788)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:785)
	at com.carrotsearch.randomizedtesting.RandomizedContext.closeResources(RandomizedContext.java:225)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.afterAlways(RandomizedRunner.java:801)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	... 1 more


FAILED:  org.apache.solr.schema.TestManagedSchemaAPI.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.schema.TestManagedSchemaAPI: 
   1) Thread[id=16823, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestManagedSchemaAPI]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:156)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:89)
        at app//org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:306)
        at app//org.apache.solr.rest.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream(ManagedResourceStorage.java:264)
        at app//org.apache.solr.rest.ManagedResourceStorage.load(ManagedResourceStorage.java:500)
        at app//org.apache.solr.rest.RestManager$RestManagerManagedResource.reloadFromStorage(RestManager.java:426)
        at app//org.apache.solr.rest.ManagedResource.loadManagedDataAndNotify(ManagedResource.java:84)
        at app//org.apache.solr.rest.RestManager.init(RestManager.java:584)
        at app//org.apache.solr.core.SolrCore.initRestManager(SolrCore.java:2969)
        at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1057)
        at app//org.apache.solr.core.SolrCore.reload(SolrCore.java:744)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1690)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3134)
        at app//org.apache.solr.core.SolrCore$$Lambda$719/0x000000080078ec40.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2499)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1284/0x0000000800cf5840.run(Unknown Source)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   2) Thread[id=16836, name=searcherExecutor-10982-thread-1-processing-127.0.0.1:62546_solr, state=WAITING, group=TGRP-TestManagedSchemaAPI]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:885)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1039)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1345)
        at java.base@11.0.13/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:232)
        at app//org.apache.solr.core.SolrCore.lambda$new$2(SolrCore.java:1048)
        at app//org.apache.solr.core.SolrCore$$Lambda$695/0x000000080073f840.call(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.FutureTask.run(FutureTask.java:264)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$247/0x000000080048fc40.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.schema.TestManagedSchemaAPI: 
   1) Thread[id=16823, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestManagedSchemaAPI]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:156)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:89)
        at app//org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:306)
        at app//org.apache.solr.rest.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream(ManagedResourceStorage.java:264)
        at app//org.apache.solr.rest.ManagedResourceStorage.load(ManagedResourceStorage.java:500)
        at app//org.apache.solr.rest.RestManager$RestManagerManagedResource.reloadFromStorage(RestManager.java:426)
        at app//org.apache.solr.rest.ManagedResource.loadManagedDataAndNotify(ManagedResource.java:84)
        at app//org.apache.solr.rest.RestManager.init(RestManager.java:584)
        at app//org.apache.solr.core.SolrCore.initRestManager(SolrCore.java:2969)
        at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1057)
        at app//org.apache.solr.core.SolrCore.reload(SolrCore.java:744)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1690)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3134)
        at app//org.apache.solr.core.SolrCore$$Lambda$719/0x000000080078ec40.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2499)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1284/0x0000000800cf5840.run(Unknown Source)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   2) Thread[id=16836, name=searcherExecutor-10982-thread-1-processing-127.0.0.1:62546_solr, state=WAITING, group=TGRP-TestManagedSchemaAPI]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:885)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1039)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1345)
        at java.base@11.0.13/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:232)
        at app//org.apache.solr.core.SolrCore.lambda$new$2(SolrCore.java:1048)
        at app//org.apache.solr.core.SolrCore$$Lambda$695/0x000000080073f840.call(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.FutureTask.run(FutureTask.java:264)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$247/0x000000080048fc40.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
	at __randomizedtesting.SeedInfo.seed([6C16A6295FEF3291]:0)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
java.security.AccessControlException: access denied ("java.io.FilePermission" "/Users/jenkins/workspace/Solr-main-MacOSX/solr/core/build/resources/test/solr/filestore" "write")

Stack Trace:
java.security.AccessControlException: access denied ("java.io.FilePermission" "/Users/jenkins/workspace/Solr-main-MacOSX/solr/core/build/resources/test/solr/filestore" "write")
	at __randomizedtesting.SeedInfo.seed([6C16A6295FEF3291]:0)
	at java.base/java.security.AccessControlContext.checkPermission(AccessControlContext.java:472)
	at java.base/java.security.AccessController.checkPermission(AccessController.java:897)
	at java.base/java.lang.SecurityManager.checkPermission(SecurityManager.java:322)
	at java.base/java.lang.SecurityManager.checkWrite(SecurityManager.java:752)
	at java.base/sun.nio.fs.UnixPath.checkWrite(UnixPath.java:824)
	at java.base/sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:381)
	at java.base/java.nio.file.Files.createDirectory(Files.java:690)
	at java.base/java.nio.file.Files.createAndCheckIsDirectory(Files.java:797)
	at java.base/java.nio.file.Files.createDirectories(Files.java:743)
	at org.apache.solr.filestore.DistribPackageStore.getPackageStoreDirPath(DistribPackageStore.java:584)
	at org.apache.solr.filestore.DistribPackageStore.ensurePackageStoreDir(DistribPackageStore.java:567)
	at org.apache.solr.filestore.DistribPackageStore.<init>(DistribPackageStore.java:83)
	at org.apache.solr.filestore.PackageStoreAPI.<init>(PackageStoreAPI.java:76)
	at org.apache.solr.core.CoreContainer.load(CoreContainer.java:725)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:184)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:156)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:162)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:112)
	at org.apache.solr.SolrTestCaseJ4.createCore(SolrTestCaseJ4.java:813)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:803)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:650)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:639)
	at org.apache.solr.cloud.AbstractZkTestCase.azt_beforeClass(AbstractZkTestCase.java:68)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:882)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:829)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 8 threads leaked from SUITE scope at org.apache.solr.rest.TestManagedResourceStorage: 
   1) Thread[id=25473, name=zkConnectionManagerCallback-16364-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   2) Thread[id=25476, name=zkCallback-16363-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   3) Thread[id=25475, name=OverseerCollectionConfigSetProcessor-72091872008601603-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:271)
        at app//org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:258)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   4) Thread[id=25464, name=httpShardExecutor-16357-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$262/0x000000080042ac40.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$218/0x00000008003f7040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   5) Thread[id=25467, name=h2sc-16359-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$262/0x000000080042ac40.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$218/0x00000008003f7040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   6) Thread[id=25471, name=SUITE-TestManagedResourceStorage-seed#[6C16A6295FEF3291]-worker-SendThread(127.0.0.1:54714), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:214)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1380)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanAndNotifyState(ClientCnxn.java:1321)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1299)
   7) Thread[id=25472, name=SUITE-TestManagedResourceStorage-seed#[6C16A6295FEF3291]-worker-EventThread, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:551)
   8) Thread[id=25474, name=OverseerStateUpdate-72091872008601603-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:294)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 8 threads leaked from SUITE scope at org.apache.solr.rest.TestManagedResourceStorage: 
   1) Thread[id=25473, name=zkConnectionManagerCallback-16364-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   2) Thread[id=25476, name=zkCallback-16363-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   3) Thread[id=25475, name=OverseerCollectionConfigSetProcessor-72091872008601603-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:271)
        at app//org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:258)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   4) Thread[id=25464, name=httpShardExecutor-16357-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$262/0x000000080042ac40.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$218/0x00000008003f7040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   5) Thread[id=25467, name=h2sc-16359-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$262/0x000000080042ac40.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$218/0x00000008003f7040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   6) Thread[id=25471, name=SUITE-TestManagedResourceStorage-seed#[6C16A6295FEF3291]-worker-SendThread(127.0.0.1:54714), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:214)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1380)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanAndNotifyState(ClientCnxn.java:1321)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1299)
   7) Thread[id=25472, name=SUITE-TestManagedResourceStorage-seed#[6C16A6295FEF3291]-worker-EventThread, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:551)
   8) Thread[id=25474, name=OverseerStateUpdate-72091872008601603-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:294)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
	at __randomizedtesting.SeedInfo.seed([6C16A6295FEF3291]:0)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=25473, name=zkConnectionManagerCallback-16364-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   2) Thread[id=25476, name=zkCallback-16363-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   3) Thread[id=25464, name=httpShardExecutor-16357-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$262/0x000000080042ac40.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$218/0x00000008003f7040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   4) Thread[id=25467, name=h2sc-16359-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$262/0x000000080042ac40.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$218/0x00000008003f7040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   5) Thread[id=25471, name=SUITE-TestManagedResourceStorage-seed#[6C16A6295FEF3291]-worker-SendThread(127.0.0.1:54714), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:362)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1197)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=25473, name=zkConnectionManagerCallback-16364-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   2) Thread[id=25476, name=zkCallback-16363-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   3) Thread[id=25464, name=httpShardExecutor-16357-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$262/0x000000080042ac40.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$218/0x00000008003f7040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   4) Thread[id=25467, name=h2sc-16359-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$262/0x000000080042ac40.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$218/0x00000008003f7040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   5) Thread[id=25471, name=SUITE-TestManagedResourceStorage-seed#[6C16A6295FEF3291]-worker-SendThread(127.0.0.1:54714), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:362)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1197)
	at __randomizedtesting.SeedInfo.seed([6C16A6295FEF3291]:0)

[JENKINS] Solr-main-MacOSX (64bit/jdk-16.0.2) - Build # 670 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/670/
Java: 64bit/jdk-16.0.2 -XX:+UseCompressedOops -XX:+UseParallelGC

4 tests failed.
FAILED:  org.apache.solr.client.solrj.impl.HttpSolrClientConPoolTest.classMethod

Error Message:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).

Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
	at __randomizedtesting.SeedInfo.seed([417A0D0CD0CF525]:0)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
java.security.AccessControlException: access denied ("java.io.FilePermission" "/Users/jenkins/workspace/Solr-main-MacOSX/solr/core/build/resources/test/solr/filestore" "write")

Stack Trace:
java.security.AccessControlException: access denied ("java.io.FilePermission" "/Users/jenkins/workspace/Solr-main-MacOSX/solr/core/build/resources/test/solr/filestore" "write")
	at __randomizedtesting.SeedInfo.seed([417A0D0CD0CF525]:0)
	at java.base/java.security.AccessControlContext.checkPermission(AccessControlContext.java:472)
	at java.base/java.security.AccessController.checkPermission(AccessController.java:1036)
	at java.base/java.lang.SecurityManager.checkPermission(SecurityManager.java:408)
	at java.base/java.lang.SecurityManager.checkWrite(SecurityManager.java:838)
	at java.base/sun.nio.fs.UnixPath.checkWrite(UnixPath.java:816)
	at java.base/sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:388)
	at java.base/java.nio.file.Files.createDirectory(Files.java:694)
	at java.base/java.nio.file.Files.createAndCheckIsDirectory(Files.java:801)
	at java.base/java.nio.file.Files.createDirectories(Files.java:747)
	at org.apache.solr.filestore.DistribPackageStore.getPackageStoreDirPath(DistribPackageStore.java:584)
	at org.apache.solr.filestore.DistribPackageStore.ensurePackageStoreDir(DistribPackageStore.java:567)
	at org.apache.solr.filestore.DistribPackageStore.<init>(DistribPackageStore.java:83)
	at org.apache.solr.filestore.PackageStoreAPI.<init>(PackageStoreAPI.java:76)
	at org.apache.solr.core.CoreContainer.load(CoreContainer.java:725)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:184)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:156)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:162)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:112)
	at org.apache.solr.SolrTestCaseJ4.createCore(SolrTestCaseJ4.java:813)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:803)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:650)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:639)
	at org.apache.solr.cloud.AbstractZkTestCase.azt_beforeClass(AbstractZkTestCase.java:68)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:78)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:567)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:882)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:831)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 8 threads leaked from SUITE scope at org.apache.solr.rest.TestManagedResourceStorage: 
   1) Thread[id=24696, name=zkCallback-16012-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   2) Thread[id=24687, name=h2sc-16008-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$271/0x0000000800f261f8.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$234/0x0000000800ed9af0.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   3) Thread[id=24691, name=SUITE-TestManagedResourceStorage-seed#[417A0D0CD0CF525]-worker-SendThread(127.0.0.1:65181), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:214)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1380)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanAndNotifyState(ClientCnxn.java:1321)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1299)
   4) Thread[id=24684, name=httpShardExecutor-16006-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$271/0x0000000800f261f8.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$234/0x0000000800ed9af0.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   5) Thread[id=24695, name=OverseerCollectionConfigSetProcessor-72090793197240323-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:271)
        at app//org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:258)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   6) Thread[id=24693, name=zkConnectionManagerCallback-16013-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   7) Thread[id=24692, name=SUITE-TestManagedResourceStorage-seed#[417A0D0CD0CF525]-worker-EventThread, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:551)
   8) Thread[id=24694, name=OverseerStateUpdate-72090793197240323-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:294)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 8 threads leaked from SUITE scope at org.apache.solr.rest.TestManagedResourceStorage: 
   1) Thread[id=24696, name=zkCallback-16012-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   2) Thread[id=24687, name=h2sc-16008-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$271/0x0000000800f261f8.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$234/0x0000000800ed9af0.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   3) Thread[id=24691, name=SUITE-TestManagedResourceStorage-seed#[417A0D0CD0CF525]-worker-SendThread(127.0.0.1:65181), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:214)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1380)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanAndNotifyState(ClientCnxn.java:1321)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1299)
   4) Thread[id=24684, name=httpShardExecutor-16006-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$271/0x0000000800f261f8.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$234/0x0000000800ed9af0.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   5) Thread[id=24695, name=OverseerCollectionConfigSetProcessor-72090793197240323-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:271)
        at app//org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:258)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   6) Thread[id=24693, name=zkConnectionManagerCallback-16013-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   7) Thread[id=24692, name=SUITE-TestManagedResourceStorage-seed#[417A0D0CD0CF525]-worker-EventThread, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:551)
   8) Thread[id=24694, name=OverseerStateUpdate-72090793197240323-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:294)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
	at __randomizedtesting.SeedInfo.seed([417A0D0CD0CF525]:0)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=24696, name=zkCallback-16012-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   2) Thread[id=24687, name=h2sc-16008-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$271/0x0000000800f261f8.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$234/0x0000000800ed9af0.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   3) Thread[id=24691, name=SUITE-TestManagedResourceStorage-seed#[417A0D0CD0CF525]-worker-SendThread(127.0.0.1:65181), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:362)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1197)
   4) Thread[id=24684, name=httpShardExecutor-16006-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$271/0x0000000800f261f8.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$234/0x0000000800ed9af0.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   5) Thread[id=24693, name=zkConnectionManagerCallback-16013-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=24696, name=zkCallback-16012-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   2) Thread[id=24687, name=h2sc-16008-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$271/0x0000000800f261f8.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$234/0x0000000800ed9af0.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   3) Thread[id=24691, name=SUITE-TestManagedResourceStorage-seed#[417A0D0CD0CF525]-worker-SendThread(127.0.0.1:65181), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:362)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1197)
   4) Thread[id=24684, name=httpShardExecutor-16006-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$271/0x0000000800f261f8.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$234/0x0000000800ed9af0.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   5) Thread[id=24693, name=zkConnectionManagerCallback-16013-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
	at __randomizedtesting.SeedInfo.seed([417A0D0CD0CF525]:0)

[JENKINS] Solr-main-MacOSX (64bit/jdk-17.0.2) - Build # 669 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/669/
Java: 64bit/jdk-17.0.2 -XX:-UseCompressedOops -XX:+UseShenandoahGC

9 tests failed.
FAILED:  org.apache.solr.cloud.HttpPartitionOnCommitTest.test

Error Message:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:63653/cenk: Underlying core creation failed while creating collection: control_collection

Stack Trace:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:63653/cenk: Underlying core creation failed while creating collection: control_collection
	at __randomizedtesting.SeedInfo.seed([7079305F4E95F920:F82D0F85E06994D8]:0)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:668)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:239)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:222)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:389)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:359)
	at app//org.apache.solr.client.solrj.impl.BaseCloudSolrClient.sendRequest(BaseCloudSolrClient.java:1134)
	at app//org.apache.solr.client.solrj.impl.BaseCloudSolrClient.requestWithRetryOnStaleState(BaseCloudSolrClient.java:898)
	at app//org.apache.solr.client.solrj.impl.BaseCloudSolrClient.request(BaseCloudSolrClient.java:832)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:226)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:243)
	at app//org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:339)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1079)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1052)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)


FAILED:  org.apache.solr.schema.ManagedSchemaRoundRobinCloudTest.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=13758, name=qtp1187934653-13758, state=RUNNABLE, group=TGRP-ManagedSchemaRoundRobinCloudTest], registration stack trace below.

Stack Trace:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=13758, name=qtp1187934653-13758, state=RUNNABLE, group=TGRP-ManagedSchemaRoundRobinCloudTest], registration stack trace below.
	at java.base@17.0.2/java.lang.Thread.getStackTrace(Thread.java:1610)
	at app//com.carrotsearch.randomizedtesting.RandomizedContext.closeAtEnd(RandomizedContext.java:157)
	at app//org.apache.lucene.util.LuceneTestCase.closeAfterSuite(LuceneTestCase.java:798)
	at app//org.apache.lucene.util.LuceneTestCase.wrapDirectory(LuceneTestCase.java:1504)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1368)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1350)
	at app//org.apache.solr.core.MockDirectoryFactory.create(MockDirectoryFactory.java:49)
	at app//org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:360)
	at app//org.apache.solr.core.SolrCore.initSnapshotMetaDataManager(SolrCore.java:582)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1026)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:961)
	at app//org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1423)
	at app//org.apache.solr.core.CoreContainer.create(CoreContainer.java:1297)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:345)
	at app//org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:423)
	at app//org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:197)
	at app//org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:214)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:866)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:830)
	at app//org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:542)
	at app//org.apache.solr.servlet.SolrDispatchFilter.dispatch(SolrDispatchFilter.java:218)
	at app//org.apache.solr.servlet.SolrDispatchFilter.lambda$doFilter$0(SolrDispatchFilter.java:189)
	at app//org.apache.solr.servlet.ServletUtils.traceHttpRequestExecution2(ServletUtils.java:232)
	at app//org.apache.solr.servlet.ServletUtils.rateLimitRequest(ServletUtils.java:208)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:187)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:171)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:189)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1434)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
	at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)
	at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1349)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
	at app//org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:713)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.server.Server.handle(Server.java:516)
	at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:400)
	at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:645)
	at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:392)
	at app//org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:352)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
	at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
Caused by: java.lang.AssertionError: Directory not closed: MockDirectoryWrapper(MMapDirectory@/Users/jenkins/workspace/Solr-main-MacOSX/solr/core/build/tmp/tests-tmp/solr.schema.ManagedSchemaRoundRobinCloudTest_7079305F4E95F920-001/index-MMapDirectory-002 lockFactory=org.apache.lucene.store.NativeFSLockFactory@b557862)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.lucene.util.CloseableDirectory.close(CloseableDirectory.java:43)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:788)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:785)
	at com.carrotsearch.randomizedtesting.RandomizedContext.closeResources(RandomizedContext.java:225)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.afterAlways(RandomizedRunner.java:801)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	... 1 more


FAILED:  org.apache.solr.schema.ManagedSchemaRoundRobinCloudTest.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=13758, name=qtp1187934653-13758, state=RUNNABLE, group=TGRP-ManagedSchemaRoundRobinCloudTest], registration stack trace below.

Stack Trace:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=13758, name=qtp1187934653-13758, state=RUNNABLE, group=TGRP-ManagedSchemaRoundRobinCloudTest], registration stack trace below.
	at java.base@17.0.2/java.lang.Thread.getStackTrace(Thread.java:1610)
	at app//com.carrotsearch.randomizedtesting.RandomizedContext.closeAtEnd(RandomizedContext.java:157)
	at app//org.apache.lucene.util.LuceneTestCase.closeAfterSuite(LuceneTestCase.java:798)
	at app//org.apache.lucene.util.LuceneTestCase.wrapDirectory(LuceneTestCase.java:1504)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1368)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1350)
	at app//org.apache.solr.core.MockDirectoryFactory.create(MockDirectoryFactory.java:49)
	at app//org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:360)
	at app//org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:404)
	at app//org.apache.solr.core.SolrCore.initIndex(SolrCore.java:817)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1030)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:961)
	at app//org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1423)
	at app//org.apache.solr.core.CoreContainer.create(CoreContainer.java:1297)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:345)
	at app//org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:423)
	at app//org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:197)
	at app//org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:214)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:866)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:830)
	at app//org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:542)
	at app//org.apache.solr.servlet.SolrDispatchFilter.dispatch(SolrDispatchFilter.java:218)
	at app//org.apache.solr.servlet.SolrDispatchFilter.lambda$doFilter$0(SolrDispatchFilter.java:189)
	at app//org.apache.solr.servlet.ServletUtils.traceHttpRequestExecution2(ServletUtils.java:232)
	at app//org.apache.solr.servlet.ServletUtils.rateLimitRequest(ServletUtils.java:208)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:187)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:171)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:189)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1434)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
	at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)
	at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1349)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
	at app//org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:713)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.server.Server.handle(Server.java:516)
	at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:400)
	at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:645)
	at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:392)
	at app//org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:352)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
	at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
Caused by: java.lang.AssertionError: Directory not closed: MockDirectoryWrapper(NIOFSDirectory@/Users/jenkins/workspace/Solr-main-MacOSX/solr/core/build/tmp/tests-tmp/solr.schema.ManagedSchemaRoundRobinCloudTest_7079305F4E95F920-001/index-NIOFSDirectory-002 lockFactory=org.apache.lucene.store.NativeFSLockFactory@b557862)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.lucene.util.CloseableDirectory.close(CloseableDirectory.java:43)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:788)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:785)
	at com.carrotsearch.randomizedtesting.RandomizedContext.closeResources(RandomizedContext.java:225)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.afterAlways(RandomizedRunner.java:801)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	... 1 more


FAILED:  org.apache.solr.schema.ManagedSchemaRoundRobinCloudTest.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=13758, name=qtp1187934653-13758, state=RUNNABLE, group=TGRP-ManagedSchemaRoundRobinCloudTest], registration stack trace below.

Stack Trace:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=13758, name=qtp1187934653-13758, state=RUNNABLE, group=TGRP-ManagedSchemaRoundRobinCloudTest], registration stack trace below.
	at java.base@17.0.2/java.lang.Thread.getStackTrace(Thread.java:1610)
	at app//com.carrotsearch.randomizedtesting.RandomizedContext.closeAtEnd(RandomizedContext.java:157)
	at app//org.apache.lucene.util.LuceneTestCase.closeAfterSuite(LuceneTestCase.java:798)
	at app//org.apache.lucene.util.LuceneTestCase.wrapDirectory(LuceneTestCase.java:1504)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1368)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1350)
	at app//org.apache.solr.core.MockDirectoryFactory.create(MockDirectoryFactory.java:49)
	at app//org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:360)
	at app//org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:98)
	at app//org.apache.solr.core.SolrCore.initIndex(SolrCore.java:849)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1030)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:961)
	at app//org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1423)
	at app//org.apache.solr.core.CoreContainer.create(CoreContainer.java:1297)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:345)
	at app//org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:423)
	at app//org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:197)
	at app//org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:214)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:866)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:830)
	at app//org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:542)
	at app//org.apache.solr.servlet.SolrDispatchFilter.dispatch(SolrDispatchFilter.java:218)
	at app//org.apache.solr.servlet.SolrDispatchFilter.lambda$doFilter$0(SolrDispatchFilter.java:189)
	at app//org.apache.solr.servlet.ServletUtils.traceHttpRequestExecution2(ServletUtils.java:232)
	at app//org.apache.solr.servlet.ServletUtils.rateLimitRequest(ServletUtils.java:208)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:187)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:171)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:189)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1434)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
	at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)
	at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1349)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
	at app//org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:713)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.server.Server.handle(Server.java:516)
	at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:400)
	at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:645)
	at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:392)
	at app//org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:352)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
	at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
Caused by: java.lang.AssertionError: Directory not closed: MockDirectoryWrapper(ByteBuffersDirectory@1567fe6c lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@35da695c)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.lucene.util.CloseableDirectory.close(CloseableDirectory.java:43)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:788)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:785)
	at com.carrotsearch.randomizedtesting.RandomizedContext.closeResources(RandomizedContext.java:225)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.afterAlways(RandomizedRunner.java:801)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	... 1 more


FAILED:  org.apache.solr.schema.ManagedSchemaRoundRobinCloudTest.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.schema.ManagedSchemaRoundRobinCloudTest: 
   1) Thread[id=13902, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-ManagedSchemaRoundRobinCloudTest]
        at java.base@17.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:156)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:89)
        at app//org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:306)
        at app//org.apache.solr.rest.ManagedResourceStorage$ZooKeeperStorageIO.configure(ManagedResourceStorage.java:232)
        at app//org.apache.solr.rest.ManagedResourceStorage.newStorageIO(ManagedResourceStorage.java:144)
        at app//org.apache.solr.core.SolrCore.initRestManager(SolrCore.java:2968)
        at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1057)
        at app//org.apache.solr.core.SolrCore.reload(SolrCore.java:744)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1690)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3134)
        at app//org.apache.solr.core.SolrCore$$Lambda$1074/0x000000080150d000.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2499)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1094/0x000000080156fd58.run(Unknown Source)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   2) Thread[id=13919, name=searcherExecutor-8362-thread-1-processing-127.0.0.1:51193_solr, state=WAITING, group=TGRP-ManagedSchemaRoundRobinCloudTest]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
        at java.base@17.0.2/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:230)
        at app//org.apache.solr.core.SolrCore.lambda$new$2(SolrCore.java:1048)
        at app//org.apache.solr.core.SolrCore$$Lambda$470/0x00000008010db250.call(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.FutureTask.run(FutureTask.java:264)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$258/0x0000000800f8fdc0.run(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.schema.ManagedSchemaRoundRobinCloudTest: 
   1) Thread[id=13902, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-ManagedSchemaRoundRobinCloudTest]
        at java.base@17.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:156)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:89)
        at app//org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:306)
        at app//org.apache.solr.rest.ManagedResourceStorage$ZooKeeperStorageIO.configure(ManagedResourceStorage.java:232)
        at app//org.apache.solr.rest.ManagedResourceStorage.newStorageIO(ManagedResourceStorage.java:144)
        at app//org.apache.solr.core.SolrCore.initRestManager(SolrCore.java:2968)
        at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1057)
        at app//org.apache.solr.core.SolrCore.reload(SolrCore.java:744)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1690)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3134)
        at app//org.apache.solr.core.SolrCore$$Lambda$1074/0x000000080150d000.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2499)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1094/0x000000080156fd58.run(Unknown Source)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   2) Thread[id=13919, name=searcherExecutor-8362-thread-1-processing-127.0.0.1:51193_solr, state=WAITING, group=TGRP-ManagedSchemaRoundRobinCloudTest]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
        at java.base@17.0.2/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:230)
        at app//org.apache.solr.core.SolrCore.lambda$new$2(SolrCore.java:1048)
        at app//org.apache.solr.core.SolrCore$$Lambda$470/0x00000008010db250.call(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.FutureTask.run(FutureTask.java:264)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$258/0x0000000800f8fdc0.run(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
	at __randomizedtesting.SeedInfo.seed([7079305F4E95F920]:0)


FAILED:  org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=5123, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@17.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:156)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:89)
        at app//org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:306)
        at app//org.apache.solr.cloud.ZkController.pathExists(ZkController.java:1139)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:80)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getSnowballWordSet(AbstractAnalysisFactory.java:321)
        at app//org.apache.lucene.analysis.en.AbstractWordsFileFilterFactory.inform(AbstractWordsFileFilterFactory.java:90)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:723)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1341)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1194)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1179)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:615)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:177)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:107)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:244)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:314)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:241)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$632/0x00000008011d0440.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:46)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:239)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1686)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3134)
        at app//org.apache.solr.core.SolrCore$$Lambda$755/0x0000000801326608.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2499)
        at app//org.apache.solr.cloud.ZkController$$Lambda$886/0x00000008013bf138.run(Unknown Source)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=5123, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@17.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:156)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:89)
        at app//org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:306)
        at app//org.apache.solr.cloud.ZkController.pathExists(ZkController.java:1139)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:80)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getSnowballWordSet(AbstractAnalysisFactory.java:321)
        at app//org.apache.lucene.analysis.en.AbstractWordsFileFilterFactory.inform(AbstractWordsFileFilterFactory.java:90)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:723)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1341)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1194)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1179)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:615)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:177)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:107)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:244)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:314)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:241)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$632/0x00000008011d0440.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:46)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:239)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1686)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3134)
        at app//org.apache.solr.core.SolrCore$$Lambda$755/0x0000000801326608.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2499)
        at app//org.apache.solr.cloud.ZkController$$Lambda$886/0x00000008013bf138.run(Unknown Source)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
	at __randomizedtesting.SeedInfo.seed([7079305F4E95F920]:0)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
java.security.AccessControlException: access denied ("java.io.FilePermission" "/Users/jenkins/workspace/Solr-main-MacOSX/solr/core/build/resources/test/solr/filestore" "write")

Stack Trace:
java.security.AccessControlException: access denied ("java.io.FilePermission" "/Users/jenkins/workspace/Solr-main-MacOSX/solr/core/build/resources/test/solr/filestore" "write")
	at __randomizedtesting.SeedInfo.seed([7079305F4E95F920]:0)
	at java.base/java.security.AccessControlContext.checkPermission(AccessControlContext.java:485)
	at java.base/java.security.AccessController.checkPermission(AccessController.java:1068)
	at java.base/java.lang.SecurityManager.checkPermission(SecurityManager.java:416)
	at java.base/java.lang.SecurityManager.checkWrite(SecurityManager.java:847)
	at java.base/sun.nio.fs.UnixPath.checkWrite(UnixPath.java:787)
	at java.base/sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:389)
	at java.base/java.nio.file.Files.createDirectory(Files.java:700)
	at java.base/java.nio.file.Files.createAndCheckIsDirectory(Files.java:807)
	at java.base/java.nio.file.Files.createDirectories(Files.java:753)
	at org.apache.solr.filestore.DistribPackageStore.getPackageStoreDirPath(DistribPackageStore.java:584)
	at org.apache.solr.filestore.DistribPackageStore.ensurePackageStoreDir(DistribPackageStore.java:567)
	at org.apache.solr.filestore.DistribPackageStore.<init>(DistribPackageStore.java:83)
	at org.apache.solr.filestore.PackageStoreAPI.<init>(PackageStoreAPI.java:76)
	at org.apache.solr.core.CoreContainer.load(CoreContainer.java:725)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:184)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:156)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:162)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:112)
	at org.apache.solr.SolrTestCaseJ4.createCore(SolrTestCaseJ4.java:813)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:803)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:650)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:639)
	at org.apache.solr.cloud.AbstractZkTestCase.azt_beforeClass(AbstractZkTestCase.java:68)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:568)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:882)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:833)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 8 threads leaked from SUITE scope at org.apache.solr.rest.TestManagedResourceStorage: 
   1) Thread[id=26507, name=OverseerCollectionConfigSetProcessor-72089758127292419-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:271)
        at app//org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:258)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   2) Thread[id=26503, name=SUITE-TestManagedResourceStorage-seed#[7079305F4E95F920]-worker-SendThread(127.0.0.1:57800), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:214)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1380)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanAndNotifyState(ClientCnxn.java:1321)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1299)
   3) Thread[id=26504, name=SUITE-TestManagedResourceStorage-seed#[7079305F4E95F920]-worker-EventThread, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3463)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3434)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at java.base@17.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:551)
   4) Thread[id=26499, name=h2sc-17217-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@17.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$282/0x0000000800f5c5f8.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$238/0x0000000800f08ec0.run(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   5) Thread[id=26496, name=httpShardExecutor-17215-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@17.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$282/0x0000000800f5c5f8.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$238/0x0000000800f08ec0.run(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   6) Thread[id=26508, name=zkCallback-17221-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@17.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   7) Thread[id=26505, name=zkConnectionManagerCallback-17222-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3463)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3434)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at java.base@17.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   8) Thread[id=26506, name=OverseerStateUpdate-72089758127292419-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:294)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 8 threads leaked from SUITE scope at org.apache.solr.rest.TestManagedResourceStorage: 
   1) Thread[id=26507, name=OverseerCollectionConfigSetProcessor-72089758127292419-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:271)
        at app//org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:258)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   2) Thread[id=26503, name=SUITE-TestManagedResourceStorage-seed#[7079305F4E95F920]-worker-SendThread(127.0.0.1:57800), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:214)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1380)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanAndNotifyState(ClientCnxn.java:1321)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1299)
   3) Thread[id=26504, name=SUITE-TestManagedResourceStorage-seed#[7079305F4E95F920]-worker-EventThread, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3463)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3434)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at java.base@17.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:551)
   4) Thread[id=26499, name=h2sc-17217-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@17.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$282/0x0000000800f5c5f8.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$238/0x0000000800f08ec0.run(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   5) Thread[id=26496, name=httpShardExecutor-17215-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@17.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$282/0x0000000800f5c5f8.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$238/0x0000000800f08ec0.run(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   6) Thread[id=26508, name=zkCallback-17221-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@17.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   7) Thread[id=26505, name=zkConnectionManagerCallback-17222-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3463)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3434)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at java.base@17.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   8) Thread[id=26506, name=OverseerStateUpdate-72089758127292419-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:294)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
	at __randomizedtesting.SeedInfo.seed([7079305F4E95F920]:0)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=26503, name=SUITE-TestManagedResourceStorage-seed#[7079305F4E95F920]-worker-SendThread(127.0.0.1:57800), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:362)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1197)
   2) Thread[id=26499, name=h2sc-17217-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@17.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$282/0x0000000800f5c5f8.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$238/0x0000000800f08ec0.run(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   3) Thread[id=26496, name=httpShardExecutor-17215-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@17.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$282/0x0000000800f5c5f8.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$238/0x0000000800f08ec0.run(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   4) Thread[id=26508, name=zkCallback-17221-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@17.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   5) Thread[id=26505, name=zkConnectionManagerCallback-17222-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3463)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3434)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at java.base@17.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=26503, name=SUITE-TestManagedResourceStorage-seed#[7079305F4E95F920]-worker-SendThread(127.0.0.1:57800), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:362)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1197)
   2) Thread[id=26499, name=h2sc-17217-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@17.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$282/0x0000000800f5c5f8.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$238/0x0000000800f08ec0.run(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   3) Thread[id=26496, name=httpShardExecutor-17215-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@17.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$282/0x0000000800f5c5f8.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$238/0x0000000800f08ec0.run(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   4) Thread[id=26508, name=zkCallback-17221-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@17.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   5) Thread[id=26505, name=zkConnectionManagerCallback-17222-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3463)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3434)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at java.base@17.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
	at __randomizedtesting.SeedInfo.seed([7079305F4E95F920]:0)

[JENKINS] Solr-main-MacOSX (64bit/jdk-16.0.2) - Build # 668 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/668/
Java: 64bit/jdk-16.0.2 -XX:+UseCompressedOops -XX:+UseParallelGC

5 tests failed.
FAILED:  org.apache.solr.core.TestLazyCores.testDontEvictUsedCore

Error Message:
java.lang.AssertionError: long request should still be busy

Stack Trace:
java.lang.AssertionError: long request should still be busy
	at __randomizedtesting.SeedInfo.seed([703AEA97CDF8BD81:7A92B4E964DE505]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.junit.Assert.assertTrue(Assert.java:42)
	at org.apache.solr.core.TestLazyCores.testDontEvictUsedCore(TestLazyCores.java:940)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:78)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:567)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:831)
	Suppressed: junit.framework.AssertionFailedError: LogListener-INFO-1 recorded log events which were not expected & removed from the queue by the test code expected:<0> but was:<1>
		at junit.framework.Assert.fail(Assert.java:57)
		at junit.framework.Assert.failNotEquals(Assert.java:329)
		at junit.framework.Assert.assertEquals(Assert.java:78)
		at junit.framework.Assert.assertEquals(Assert.java:234)
		at org.apache.solr.util.LogListener.close(LogListener.java:230)
		at org.apache.solr.core.TestLazyCores.testDontEvictUsedCore(TestLazyCores.java:909)
		... 46 more


FAILED:  org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=6010, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:156)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:89)
        at app//org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:306)
        at app//org.apache.solr.cloud.ZkController.pathExists(ZkController.java:1139)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:80)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getSnowballWordSet(AbstractAnalysisFactory.java:321)
        at app//org.apache.lucene.analysis.en.AbstractWordsFileFilterFactory.inform(AbstractWordsFileFilterFactory.java:90)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:723)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1341)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1194)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1179)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:615)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:177)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:107)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:244)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:314)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:241)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$602/0x000000080112fd70.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:46)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:239)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1686)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3134)
        at app//org.apache.solr.core.SolrCore$$Lambda$725/0x0000000801295ba0.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2499)
        at app//org.apache.solr.cloud.ZkController$$Lambda$854/0x00000008012e5480.run(Unknown Source)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=6010, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:156)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:89)
        at app//org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:306)
        at app//org.apache.solr.cloud.ZkController.pathExists(ZkController.java:1139)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:80)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getSnowballWordSet(AbstractAnalysisFactory.java:321)
        at app//org.apache.lucene.analysis.en.AbstractWordsFileFilterFactory.inform(AbstractWordsFileFilterFactory.java:90)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:723)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1341)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1194)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1179)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:615)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:177)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:107)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:244)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:314)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:241)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$602/0x000000080112fd70.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:46)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:239)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1686)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3134)
        at app//org.apache.solr.core.SolrCore$$Lambda$725/0x0000000801295ba0.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2499)
        at app//org.apache.solr.cloud.ZkController$$Lambda$854/0x00000008012e5480.run(Unknown Source)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
	at __randomizedtesting.SeedInfo.seed([703AEA97CDF8BD81]:0)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
java.security.AccessControlException: access denied ("java.io.FilePermission" "/Users/jenkins/workspace/Solr-main-MacOSX/solr/core/build/resources/test/solr/filestore" "write")

Stack Trace:
java.security.AccessControlException: access denied ("java.io.FilePermission" "/Users/jenkins/workspace/Solr-main-MacOSX/solr/core/build/resources/test/solr/filestore" "write")
	at __randomizedtesting.SeedInfo.seed([703AEA97CDF8BD81]:0)
	at java.base/java.security.AccessControlContext.checkPermission(AccessControlContext.java:472)
	at java.base/java.security.AccessController.checkPermission(AccessController.java:1036)
	at java.base/java.lang.SecurityManager.checkPermission(SecurityManager.java:408)
	at java.base/java.lang.SecurityManager.checkWrite(SecurityManager.java:838)
	at java.base/sun.nio.fs.UnixPath.checkWrite(UnixPath.java:816)
	at java.base/sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:388)
	at java.base/java.nio.file.Files.createDirectory(Files.java:694)
	at java.base/java.nio.file.Files.createAndCheckIsDirectory(Files.java:801)
	at java.base/java.nio.file.Files.createDirectories(Files.java:747)
	at org.apache.solr.filestore.DistribPackageStore.getPackageStoreDirPath(DistribPackageStore.java:584)
	at org.apache.solr.filestore.DistribPackageStore.ensurePackageStoreDir(DistribPackageStore.java:567)
	at org.apache.solr.filestore.DistribPackageStore.<init>(DistribPackageStore.java:83)
	at org.apache.solr.filestore.PackageStoreAPI.<init>(PackageStoreAPI.java:76)
	at org.apache.solr.core.CoreContainer.load(CoreContainer.java:725)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:184)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:156)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:162)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:112)
	at org.apache.solr.SolrTestCaseJ4.createCore(SolrTestCaseJ4.java:813)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:803)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:650)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:639)
	at org.apache.solr.cloud.AbstractZkTestCase.azt_beforeClass(AbstractZkTestCase.java:68)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:78)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:567)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:882)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:831)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 8 threads leaked from SUITE scope at org.apache.solr.rest.TestManagedResourceStorage: 
   1) Thread[id=27425, name=SUITE-TestManagedResourceStorage-seed#[703AEA97CDF8BD81]-worker-EventThread, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:551)
   2) Thread[id=27429, name=zkCallback-17310-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   3) Thread[id=27428, name=OverseerCollectionConfigSetProcessor-72088974412087299-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:271)
        at app//org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:258)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   4) Thread[id=27426, name=zkConnectionManagerCallback-17311-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   5) Thread[id=27427, name=OverseerStateUpdate-72088974412087299-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:294)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   6) Thread[id=27424, name=SUITE-TestManagedResourceStorage-seed#[703AEA97CDF8BD81]-worker-SendThread(127.0.0.1:55756), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:214)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1380)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanAndNotifyState(ClientCnxn.java:1321)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1299)
   7) Thread[id=27420, name=h2sc-17306-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$271/0x0000000800f20988.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$234/0x0000000800ed4800.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   8) Thread[id=27417, name=httpShardExecutor-17304-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$271/0x0000000800f20988.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$234/0x0000000800ed4800.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 8 threads leaked from SUITE scope at org.apache.solr.rest.TestManagedResourceStorage: 
   1) Thread[id=27425, name=SUITE-TestManagedResourceStorage-seed#[703AEA97CDF8BD81]-worker-EventThread, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:551)
   2) Thread[id=27429, name=zkCallback-17310-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   3) Thread[id=27428, name=OverseerCollectionConfigSetProcessor-72088974412087299-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:271)
        at app//org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:258)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   4) Thread[id=27426, name=zkConnectionManagerCallback-17311-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   5) Thread[id=27427, name=OverseerStateUpdate-72088974412087299-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:294)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   6) Thread[id=27424, name=SUITE-TestManagedResourceStorage-seed#[703AEA97CDF8BD81]-worker-SendThread(127.0.0.1:55756), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:214)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1380)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanAndNotifyState(ClientCnxn.java:1321)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1299)
   7) Thread[id=27420, name=h2sc-17306-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$271/0x0000000800f20988.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$234/0x0000000800ed4800.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   8) Thread[id=27417, name=httpShardExecutor-17304-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$271/0x0000000800f20988.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$234/0x0000000800ed4800.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
	at __randomizedtesting.SeedInfo.seed([703AEA97CDF8BD81]:0)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=27429, name=zkCallback-17310-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   2) Thread[id=27426, name=zkConnectionManagerCallback-17311-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   3) Thread[id=27424, name=SUITE-TestManagedResourceStorage-seed#[703AEA97CDF8BD81]-worker-SendThread(127.0.0.1:55756), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:214)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1380)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanAndNotifyState(ClientCnxn.java:1321)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1299)
   4) Thread[id=27420, name=h2sc-17306-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$271/0x0000000800f20988.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$234/0x0000000800ed4800.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   5) Thread[id=27417, name=httpShardExecutor-17304-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$271/0x0000000800f20988.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$234/0x0000000800ed4800.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=27429, name=zkCallback-17310-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   2) Thread[id=27426, name=zkConnectionManagerCallback-17311-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   3) Thread[id=27424, name=SUITE-TestManagedResourceStorage-seed#[703AEA97CDF8BD81]-worker-SendThread(127.0.0.1:55756), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:214)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1380)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanAndNotifyState(ClientCnxn.java:1321)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1299)
   4) Thread[id=27420, name=h2sc-17306-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$271/0x0000000800f20988.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$234/0x0000000800ed4800.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   5) Thread[id=27417, name=httpShardExecutor-17304-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$271/0x0000000800f20988.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$234/0x0000000800ed4800.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
	at __randomizedtesting.SeedInfo.seed([703AEA97CDF8BD81]:0)

[JENKINS] Solr-main-MacOSX (64bit/jdk-11.0.13) - Build # 667 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/667/
Java: 64bit/jdk-11.0.13 -XX:-UseCompressedOops -XX:+UseSerialGC

6 tests failed.
FAILED:  org.apache.solr.handler.TestSolrConfigHandlerCloud.test

Error Message:
java.lang.AssertionError: {
  "responseHeader":{
    "status":500,
    "QTime":30104},
  "errorMessages":["4 out of 5 the property overlay to be of version 12 within 30 seconds! Failed cores: [https://127.0.0.1:51366/collection1_shard2_replica_n2/, https://127.0.0.1:51389/collection1_shard2_replica_n5/, https://127.0.0.1:51375/collection1_shard1_replica_n1/, https://127.0.0.1:51395/collection1_shard1_replica_n6/]\n"],
  "WARNING":"This response format is experimental.  It is likely to change in the future.",
  "error":{
    "metadata":[
      "error-class","org.apache.solr.common.SolrException",
      "root-error-class","org.apache.solr.common.SolrException"],
    "msg":"4 out of 5 the property overlay to be of version 12 within 30 seconds! Failed cores: [https://127.0.0.1:51366/collection1_shard2_replica_n2/, https://127.0.0.1:51389/collection1_shard2_replica_n5/, https://127.0.0.1:51375/collection1_shard1_replica_n1/, https://127.0.0.1:51395/collection1_shard1_replica_n6/]",
    "trace":"org.apache.solr.common.SolrException: 4 out of 5 the property overlay to be of version 12 within 30 seconds! Failed cores: [https://127.0.0.1:51366/collection1_shard2_replica_n2/, https://127.0.0.1:51389/collection1_shard2_replica_n5/, https://127.0.0.1:51375/collection1_shard1_replica_n1/, https://127.0.0.1:51395/collection1_shard1_replica_n6/]\n\tat org.apache.solr.handler.SolrConfigHandler.waitForAllReplicasState(SolrConfigHandler.java:812)\n\tat org.apache.solr.handler.SolrConfigHandler$Command.handleCommands(SolrConfigHandler.java:534)\n\tat org.apache.solr.handler.SolrConfigHandler$Command.handlePOST(SolrConfigHandler.java:366)\n\tat org.apache.solr.handler.SolrConfigHandler.handleRequestBody(SolrConfigHandler.java:137)\n\tat org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:214)\n\tat org.apache.solr.core.SolrCore.execute(SolrCore.java:2638)\n\tat org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:824)\n\tat org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:564)\n\tat org.apache.solr.servlet.SolrDispatchFilter.dispatch(SolrDispatchFilter.java:218)\n\tat org.apache.solr.servlet.SolrDispatchFilter.lambda$doFilter$0(SolrDispatchFilter.java:189)\n\tat org.apache.solr.servlet.ServletUtils.traceHttpRequestExecution2(ServletUtils.java:232)\n\tat org.apache.solr.servlet.ServletUtils.rateLimitRequest(ServletUtils.java:208)\n\tat org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:187)\n\tat org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:171)\n\tat org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)\n\tat org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)\n\tat org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:189)\n\tat org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)\n\tat org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)\n\tat org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)\n\tat org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)\n\tat org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1434)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)\n\tat org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)\n\tat org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)\n\tat org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1349)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)\n\tat org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)\n\tat org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)\n\tat org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:713)\n\tat org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)\n\tat org.eclipse.jetty.server.Server.handle(Server.java:516)\n\tat org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:400)\n\tat org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:645)\n\tat org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:392)\n\tat org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:277)\n\tat org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311)\n\tat org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:105)\n\tat org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:555)\n\tat org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:410)\n\tat org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:164)\n\tat org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:105)\n\tat org.eclipse.jetty.io.ChannelEndPoint$1.run(ChannelEndPoint.java:104)\n\tat org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)\n\tat org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)\n\tat java.base/java.lang.Thread.run(Thread.java:829)\n",
    "code":500}}
 expected null, but was:<[4 out of 5 the property overlay to be of version 12 within 30 seconds! Failed cores: [https://127.0.0.1:51366/collection1_shard2_replica_n2/, https://127.0.0.1:51389/collection1_shard2_replica_n5/, https://127.0.0.1:51375/collection1_shard1_replica_n1/, https://127.0.0.1:51395/collection1_shard1_replica_n6/]
]>

Stack Trace:
java.lang.AssertionError: {
  "responseHeader":{
    "status":500,
    "QTime":30104},
  "errorMessages":["4 out of 5 the property overlay to be of version 12 within 30 seconds! Failed cores: [https://127.0.0.1:51366/collection1_shard2_replica_n2/, https://127.0.0.1:51389/collection1_shard2_replica_n5/, https://127.0.0.1:51375/collection1_shard1_replica_n1/, https://127.0.0.1:51395/collection1_shard1_replica_n6/]\n"],
  "WARNING":"This response format is experimental.  It is likely to change in the future.",
  "error":{
    "metadata":[
      "error-class","org.apache.solr.common.SolrException",
      "root-error-class","org.apache.solr.common.SolrException"],
    "msg":"4 out of 5 the property overlay to be of version 12 within 30 seconds! Failed cores: [https://127.0.0.1:51366/collection1_shard2_replica_n2/, https://127.0.0.1:51389/collection1_shard2_replica_n5/, https://127.0.0.1:51375/collection1_shard1_replica_n1/, https://127.0.0.1:51395/collection1_shard1_replica_n6/]",
    "trace":"org.apache.solr.common.SolrException: 4 out of 5 the property overlay to be of version 12 within 30 seconds! Failed cores: [https://127.0.0.1:51366/collection1_shard2_replica_n2/, https://127.0.0.1:51389/collection1_shard2_replica_n5/, https://127.0.0.1:51375/collection1_shard1_replica_n1/, https://127.0.0.1:51395/collection1_shard1_replica_n6/]\n\tat org.apache.solr.handler.SolrConfigHandler.waitForAllReplicasState(SolrConfigHandler.java:812)\n\tat org.apache.solr.handler.SolrConfigHandler$Command.handleCommands(SolrConfigHandler.java:534)\n\tat org.apache.solr.handler.SolrConfigHandler$Command.handlePOST(SolrConfigHandler.java:366)\n\tat org.apache.solr.handler.SolrConfigHandler.handleRequestBody(SolrConfigHandler.java:137)\n\tat org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:214)\n\tat org.apache.solr.core.SolrCore.execute(SolrCore.java:2638)\n\tat org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:824)\n\tat org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:564)\n\tat org.apache.solr.servlet.SolrDispatchFilter.dispatch(SolrDispatchFilter.java:218)\n\tat org.apache.solr.servlet.SolrDispatchFilter.lambda$doFilter$0(SolrDispatchFilter.java:189)\n\tat org.apache.solr.servlet.ServletUtils.traceHttpRequestExecution2(ServletUtils.java:232)\n\tat org.apache.solr.servlet.ServletUtils.rateLimitRequest(ServletUtils.java:208)\n\tat org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:187)\n\tat org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:171)\n\tat org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)\n\tat org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)\n\tat org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:189)\n\tat org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)\n\tat org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)\n\tat org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)\n\tat org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)\n\tat org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1434)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)\n\tat org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)\n\tat org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)\n\tat org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1349)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)\n\tat org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)\n\tat org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)\n\tat org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:713)\n\tat org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)\n\tat org.eclipse.jetty.server.Server.handle(Server.java:516)\n\tat org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:400)\n\tat org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:645)\n\tat org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:392)\n\tat org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:277)\n\tat org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311)\n\tat org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:105)\n\tat org.eclipse.jetty.io.ssl.SslConnection$DecryptedEndPoint.onFillable(SslConnection.java:555)\n\tat org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:410)\n\tat org.eclipse.jetty.io.ssl.SslConnection$2.succeeded(SslConnection.java:164)\n\tat org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:105)\n\tat org.eclipse.jetty.io.ChannelEndPoint$1.run(ChannelEndPoint.java:104)\n\tat org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)\n\tat org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)\n\tat java.base/java.lang.Thread.run(Thread.java:829)\n",
    "code":500}}
 expected null, but was:<[4 out of 5 the property overlay to be of version 12 within 30 seconds! Failed cores: [https://127.0.0.1:51366/collection1_shard2_replica_n2/, https://127.0.0.1:51389/collection1_shard2_replica_n5/, https://127.0.0.1:51375/collection1_shard1_replica_n1/, https://127.0.0.1:51395/collection1_shard1_replica_n6/]
]>
	at __randomizedtesting.SeedInfo.seed([6A465FF56A064279:E212602FC4FA2F81]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.junit.Assert.failNotNull(Assert.java:756)
	at org.junit.Assert.assertNull(Assert.java:738)
	at org.apache.solr.core.TestSolrConfigHandler.runConfigCommand(TestSolrConfigHandler.java:221)
	at org.apache.solr.core.TestSolrConfigHandler.reqhandlertests(TestSolrConfigHandler.java:417)
	at org.apache.solr.handler.TestSolrConfigHandlerCloud.testReqHandlerAPIs(TestSolrConfigHandlerCloud.java:80)
	at org.apache.solr.handler.TestSolrConfigHandlerCloud.test(TestSolrConfigHandlerCloud.java:50)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1081)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1052)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:829)


FAILED:  org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=3476, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:156)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:89)
        at app//org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:342)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:82)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getLines(AbstractAnalysisFactory.java:302)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getWordSet(AbstractAnalysisFactory.java:293)
        at app//org.apache.lucene.analysis.en.AbstractWordsFileFilterFactory.inform(AbstractWordsFileFilterFactory.java:88)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:723)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1341)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1194)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1179)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:615)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:177)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:107)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:244)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:314)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:241)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$609/0x000000011216d908.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:46)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:239)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1686)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3134)
        at app//org.apache.solr.core.SolrCore$$Lambda$732/0x000000011289c9b8.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2499)
        at app//org.apache.solr.cloud.ZkController$$Lambda$841/0x0000000112cae8b0.run(Unknown Source)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=3476, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:156)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:89)
        at app//org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:342)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:82)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getLines(AbstractAnalysisFactory.java:302)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getWordSet(AbstractAnalysisFactory.java:293)
        at app//org.apache.lucene.analysis.en.AbstractWordsFileFilterFactory.inform(AbstractWordsFileFilterFactory.java:88)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:723)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1341)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1194)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1179)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:615)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:177)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:107)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:244)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:314)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:241)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$609/0x000000011216d908.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:46)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:239)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1686)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3134)
        at app//org.apache.solr.core.SolrCore$$Lambda$732/0x000000011289c9b8.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2499)
        at app//org.apache.solr.cloud.ZkController$$Lambda$841/0x0000000112cae8b0.run(Unknown Source)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
	at __randomizedtesting.SeedInfo.seed([6A465FF56A064279]:0)


FAILED:  org.apache.solr.cloud.HttpPartitionOnCommitTest.test

Error Message:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:51970/h_i/t: Underlying core creation failed while creating collection: c8n_1x3_commits

Stack Trace:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:51970/h_i/t: Underlying core creation failed while creating collection: c8n_1x3_commits
	at __randomizedtesting.SeedInfo.seed([6A465FF56A064279:E212602FC4FA2F81]:0)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:668)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:239)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:222)
	at app//org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1290)
	at app//org.apache.solr.cloud.AbstractBasicDistributedZkTestBase.createCollection(AbstractBasicDistributedZkTestBase.java:803)
	at app//org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1747)
	at app//org.apache.solr.cloud.HttpPartitionOnCommitTest.oneShardTest(HttpPartitionOnCommitTest.java:125)
	at app//org.apache.solr.cloud.HttpPartitionOnCommitTest.test(HttpPartitionOnCommitTest.java:66)
	at java.base@11.0.13/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base@11.0.13/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base@11.0.13/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base@11.0.13/java.lang.reflect.Method.invoke(Method.java:566)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1081)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1052)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
java.security.AccessControlException: access denied ("java.io.FilePermission" "/Users/jenkins/workspace/Solr-main-MacOSX/solr/core/build/resources/test/solr/filestore" "write")

Stack Trace:
java.security.AccessControlException: access denied ("java.io.FilePermission" "/Users/jenkins/workspace/Solr-main-MacOSX/solr/core/build/resources/test/solr/filestore" "write")
	at __randomizedtesting.SeedInfo.seed([6A465FF56A064279]:0)
	at java.base/java.security.AccessControlContext.checkPermission(AccessControlContext.java:472)
	at java.base/java.security.AccessController.checkPermission(AccessController.java:897)
	at java.base/java.lang.SecurityManager.checkPermission(SecurityManager.java:322)
	at java.base/java.lang.SecurityManager.checkWrite(SecurityManager.java:752)
	at java.base/sun.nio.fs.UnixPath.checkWrite(UnixPath.java:824)
	at java.base/sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:381)
	at java.base/java.nio.file.Files.createDirectory(Files.java:690)
	at java.base/java.nio.file.Files.createAndCheckIsDirectory(Files.java:797)
	at java.base/java.nio.file.Files.createDirectories(Files.java:743)
	at org.apache.solr.filestore.DistribPackageStore.getPackageStoreDirPath(DistribPackageStore.java:584)
	at org.apache.solr.filestore.DistribPackageStore.ensurePackageStoreDir(DistribPackageStore.java:567)
	at org.apache.solr.filestore.DistribPackageStore.<init>(DistribPackageStore.java:83)
	at org.apache.solr.filestore.PackageStoreAPI.<init>(PackageStoreAPI.java:76)
	at org.apache.solr.core.CoreContainer.load(CoreContainer.java:725)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:184)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:156)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:162)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:112)
	at org.apache.solr.SolrTestCaseJ4.createCore(SolrTestCaseJ4.java:813)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:803)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:650)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:639)
	at org.apache.solr.cloud.AbstractZkTestCase.azt_beforeClass(AbstractZkTestCase.java:68)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:882)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:829)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 8 threads leaked from SUITE scope at org.apache.solr.rest.TestManagedResourceStorage: 
   1) Thread[id=23619, name=zkCallback-15393-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   2) Thread[id=23616, name=zkConnectionManagerCallback-15394-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   3) Thread[id=23618, name=OverseerCollectionConfigSetProcessor-72087667657998339-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:271)
        at app//org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:258)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   4) Thread[id=23614, name=SUITE-TestManagedResourceStorage-seed#[6A465FF56A064279]-worker-SendThread(127.0.0.1:62860), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:214)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1380)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanAndNotifyState(ClientCnxn.java:1321)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1299)
   5) Thread[id=23615, name=SUITE-TestManagedResourceStorage-seed#[6A465FF56A064279]-worker-EventThread, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:551)
   6) Thread[id=23617, name=OverseerStateUpdate-72087667657998339-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:294)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   7) Thread[id=23607, name=httpShardExecutor-15387-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$258/0x0000000110267c58.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$218/0x0000000110d50db8.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   8) Thread[id=23610, name=h2sc-15389-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$258/0x0000000110267c58.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$218/0x0000000110d50db8.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 8 threads leaked from SUITE scope at org.apache.solr.rest.TestManagedResourceStorage: 
   1) Thread[id=23619, name=zkCallback-15393-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   2) Thread[id=23616, name=zkConnectionManagerCallback-15394-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   3) Thread[id=23618, name=OverseerCollectionConfigSetProcessor-72087667657998339-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:271)
        at app//org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:258)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   4) Thread[id=23614, name=SUITE-TestManagedResourceStorage-seed#[6A465FF56A064279]-worker-SendThread(127.0.0.1:62860), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:214)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1380)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanAndNotifyState(ClientCnxn.java:1321)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1299)
   5) Thread[id=23615, name=SUITE-TestManagedResourceStorage-seed#[6A465FF56A064279]-worker-EventThread, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:551)
   6) Thread[id=23617, name=OverseerStateUpdate-72087667657998339-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:294)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   7) Thread[id=23607, name=httpShardExecutor-15387-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$258/0x0000000110267c58.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$218/0x0000000110d50db8.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   8) Thread[id=23610, name=h2sc-15389-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$258/0x0000000110267c58.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$218/0x0000000110d50db8.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
	at __randomizedtesting.SeedInfo.seed([6A465FF56A064279]:0)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=23619, name=zkCallback-15393-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   2) Thread[id=23616, name=zkConnectionManagerCallback-15394-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   3) Thread[id=23614, name=SUITE-TestManagedResourceStorage-seed#[6A465FF56A064279]-worker-SendThread(127.0.0.1:62860), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:362)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1197)
   4) Thread[id=23607, name=httpShardExecutor-15387-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$258/0x0000000110267c58.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$218/0x0000000110d50db8.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   5) Thread[id=23610, name=h2sc-15389-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$258/0x0000000110267c58.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$218/0x0000000110d50db8.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=23619, name=zkCallback-15393-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   2) Thread[id=23616, name=zkConnectionManagerCallback-15394-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   3) Thread[id=23614, name=SUITE-TestManagedResourceStorage-seed#[6A465FF56A064279]-worker-SendThread(127.0.0.1:62860), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:362)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1197)
   4) Thread[id=23607, name=httpShardExecutor-15387-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$258/0x0000000110267c58.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$218/0x0000000110d50db8.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   5) Thread[id=23610, name=h2sc-15389-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$258/0x0000000110267c58.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$218/0x0000000110d50db8.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
	at __randomizedtesting.SeedInfo.seed([6A465FF56A064279]:0)

[JENKINS] Solr-main-MacOSX (64bit/jdk-16.0.2) - Build # 666 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/666/
Java: 64bit/jdk-16.0.2 -XX:+UseCompressedOops -XX:+UseShenandoahGC

6 tests failed.
FAILED:  org.apache.solr.cloud.HttpPartitionOnCommitTest.test

Error Message:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:51667: Underlying core creation failed while creating collection: c8n_1x3_commits

Stack Trace:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:51667: Underlying core creation failed while creating collection: c8n_1x3_commits
	at __randomizedtesting.SeedInfo.seed([D6301D26529438C6:5E6422FCFC68553E]:0)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:668)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:239)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:222)
	at app//org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1290)
	at app//org.apache.solr.cloud.AbstractBasicDistributedZkTestBase.createCollection(AbstractBasicDistributedZkTestBase.java:803)
	at app//org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1747)
	at app//org.apache.solr.cloud.HttpPartitionOnCommitTest.oneShardTest(HttpPartitionOnCommitTest.java:125)
	at app//org.apache.solr.cloud.HttpPartitionOnCommitTest.test(HttpPartitionOnCommitTest.java:66)
	at java.base@16.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base@16.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:78)
	at java.base@16.0.2/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base@16.0.2/java.lang.reflect.Method.invoke(Method.java:567)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1081)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1052)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)


FAILED:  org.apache.solr.metrics.reporters.SolrJmxReporterTest.testClosedCore

Error Message:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=4598, name=Thread-1001, state=RUNNABLE, group=TGRP-SolrJmxReporterTest]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=4598, name=Thread-1001, state=RUNNABLE, group=TGRP-SolrJmxReporterTest]
	at __randomizedtesting.SeedInfo.seed([D6301D26529438C6:A14DCB06720D5A36]:0)
Caused by: java.lang.AssertionError
	at __randomizedtesting.SeedInfo.seed([D6301D26529438C6]:0)
	at org.junit.Assert.fail(Assert.java:87)
	at org.junit.Assert.assertTrue(Assert.java:42)
	at org.junit.Assert.assertNotNull(Assert.java:713)
	at org.junit.Assert.assertNotNull(Assert.java:723)
	at org.apache.solr.metrics.reporters.SolrJmxReporterTest$1.run(SolrJmxReporterTest.java:213)


FAILED:  org.apache.solr.core.TestLazyCores.testDontEvictUsedCore

Error Message:
java.lang.AssertionError

Stack Trace:
java.lang.AssertionError
	at __randomizedtesting.SeedInfo.seed([D6301D26529438C6:A1A3DCFF09216042]:0)
	at org.junit.Assert.fail(Assert.java:87)
	at org.junit.Assert.assertTrue(Assert.java:42)
	at org.junit.Assert.assertTrue(Assert.java:53)
	at org.apache.solr.core.TestLazyCores.testDontEvictUsedCore(TestLazyCores.java:937)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:78)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:567)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:831)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
java.security.AccessControlException: access denied ("java.io.FilePermission" "/Users/jenkins/workspace/Solr-main-MacOSX/solr/core/build/resources/test/solr/filestore" "write")

Stack Trace:
java.security.AccessControlException: access denied ("java.io.FilePermission" "/Users/jenkins/workspace/Solr-main-MacOSX/solr/core/build/resources/test/solr/filestore" "write")
	at __randomizedtesting.SeedInfo.seed([D6301D26529438C6]:0)
	at java.base/java.security.AccessControlContext.checkPermission(AccessControlContext.java:472)
	at java.base/java.security.AccessController.checkPermission(AccessController.java:1036)
	at java.base/java.lang.SecurityManager.checkPermission(SecurityManager.java:408)
	at java.base/java.lang.SecurityManager.checkWrite(SecurityManager.java:838)
	at java.base/sun.nio.fs.UnixPath.checkWrite(UnixPath.java:816)
	at java.base/sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:388)
	at java.base/java.nio.file.Files.createDirectory(Files.java:694)
	at java.base/java.nio.file.Files.createAndCheckIsDirectory(Files.java:801)
	at java.base/java.nio.file.Files.createDirectories(Files.java:747)
	at org.apache.solr.filestore.DistribPackageStore.getPackageStoreDirPath(DistribPackageStore.java:584)
	at org.apache.solr.filestore.DistribPackageStore.ensurePackageStoreDir(DistribPackageStore.java:567)
	at org.apache.solr.filestore.DistribPackageStore.<init>(DistribPackageStore.java:83)
	at org.apache.solr.filestore.PackageStoreAPI.<init>(PackageStoreAPI.java:76)
	at org.apache.solr.core.CoreContainer.load(CoreContainer.java:725)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:184)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:156)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:162)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:112)
	at org.apache.solr.SolrTestCaseJ4.createCore(SolrTestCaseJ4.java:813)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:803)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:650)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:639)
	at org.apache.solr.cloud.AbstractZkTestCase.azt_beforeClass(AbstractZkTestCase.java:68)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:78)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:567)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:882)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:831)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 8 threads leaked from SUITE scope at org.apache.solr.rest.TestManagedResourceStorage: 
   1) Thread[id=25341, name=SUITE-TestManagedResourceStorage-seed#[D6301D26529438C6]-worker-EventThread, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:551)
   2) Thread[id=25333, name=httpShardExecutor-16533-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$275/0x0000000800f21e00.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$238/0x0000000800ed5d78.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   3) Thread[id=25344, name=OverseerCollectionConfigSetProcessor-72086773541699587-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:271)
        at app//org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:258)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   4) Thread[id=25342, name=zkConnectionManagerCallback-16540-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   5) Thread[id=25340, name=SUITE-TestManagedResourceStorage-seed#[D6301D26529438C6]-worker-SendThread(127.0.0.1:53643), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:214)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1380)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanAndNotifyState(ClientCnxn.java:1321)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1299)
   6) Thread[id=25343, name=OverseerStateUpdate-72086773541699587-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:294)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   7) Thread[id=25336, name=h2sc-16535-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$275/0x0000000800f21e00.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$238/0x0000000800ed5d78.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   8) Thread[id=25345, name=zkCallback-16539-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 8 threads leaked from SUITE scope at org.apache.solr.rest.TestManagedResourceStorage: 
   1) Thread[id=25341, name=SUITE-TestManagedResourceStorage-seed#[D6301D26529438C6]-worker-EventThread, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:551)
   2) Thread[id=25333, name=httpShardExecutor-16533-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$275/0x0000000800f21e00.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$238/0x0000000800ed5d78.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   3) Thread[id=25344, name=OverseerCollectionConfigSetProcessor-72086773541699587-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:271)
        at app//org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:258)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   4) Thread[id=25342, name=zkConnectionManagerCallback-16540-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   5) Thread[id=25340, name=SUITE-TestManagedResourceStorage-seed#[D6301D26529438C6]-worker-SendThread(127.0.0.1:53643), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:214)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1380)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanAndNotifyState(ClientCnxn.java:1321)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1299)
   6) Thread[id=25343, name=OverseerStateUpdate-72086773541699587-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:294)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   7) Thread[id=25336, name=h2sc-16535-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$275/0x0000000800f21e00.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$238/0x0000000800ed5d78.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   8) Thread[id=25345, name=zkCallback-16539-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
	at __randomizedtesting.SeedInfo.seed([D6301D26529438C6]:0)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=25333, name=httpShardExecutor-16533-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$275/0x0000000800f21e00.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$238/0x0000000800ed5d78.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   2) Thread[id=25342, name=zkConnectionManagerCallback-16540-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   3) Thread[id=25340, name=SUITE-TestManagedResourceStorage-seed#[D6301D26529438C6]-worker-SendThread(127.0.0.1:53643), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:362)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1197)
   4) Thread[id=25336, name=h2sc-16535-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$275/0x0000000800f21e00.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$238/0x0000000800ed5d78.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   5) Thread[id=25345, name=zkCallback-16539-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=25333, name=httpShardExecutor-16533-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$275/0x0000000800f21e00.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$238/0x0000000800ed5d78.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   2) Thread[id=25342, name=zkConnectionManagerCallback-16540-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   3) Thread[id=25340, name=SUITE-TestManagedResourceStorage-seed#[D6301D26529438C6]-worker-SendThread(127.0.0.1:53643), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:362)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1197)
   4) Thread[id=25336, name=h2sc-16535-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$275/0x0000000800f21e00.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$238/0x0000000800ed5d78.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   5) Thread[id=25345, name=zkCallback-16539-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
	at __randomizedtesting.SeedInfo.seed([D6301D26529438C6]:0)

[JENKINS] Solr-main-MacOSX (64bit/jdk-17.0.2) - Build # 665 - Still unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/665/
Java: 64bit/jdk-17.0.2 -XX:+UseCompressedOops -XX:+UseShenandoahGC

6 tests failed.
FAILED:  org.apache.solr.cloud.TestRebalanceLeaders.testRebalanceLeaders

Error Message:
org.apache.solr.client.solrj.SolrServerException: Timeout occurred while waiting response from server at: https://127.0.0.1:61034/solr

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occurred while waiting response from server at: https://127.0.0.1:61034/solr
	at __randomizedtesting.SeedInfo.seed([33FE595988D408C4:2D116BA9B07F44B4]:0)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:679)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:239)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:222)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:389)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:359)
	at app//org.apache.solr.client.solrj.impl.BaseCloudSolrClient.sendRequest(BaseCloudSolrClient.java:1134)
	at app//org.apache.solr.client.solrj.impl.BaseCloudSolrClient.requestWithRetryOnStaleState(BaseCloudSolrClient.java:898)
	at app//org.apache.solr.client.solrj.impl.BaseCloudSolrClient.request(BaseCloudSolrClient.java:832)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:226)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:243)
	at app//org.apache.solr.cloud.TestRebalanceLeaders.rebalanceLeaderUsingSolrJAPI(TestRebalanceLeaders.java:373)
	at app//org.apache.solr.cloud.TestRebalanceLeaders.sendRebalanceCommand(TestRebalanceLeaders.java:279)
	at app//org.apache.solr.cloud.TestRebalanceLeaders.testRebalanceLeaders(TestRebalanceLeaders.java:145)
	at java.base@17.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base@17.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base@17.0.2/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base@17.0.2/java.lang.reflect.Method.invoke(Method.java:568)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.base/sun.nio.ch.NioSocketImpl.timedRead(NioSocketImpl.java:283)
	at java.base/sun.nio.ch.NioSocketImpl.implRead(NioSocketImpl.java:309)
	at java.base/sun.nio.ch.NioSocketImpl.read(NioSocketImpl.java:350)
	at java.base/sun.nio.ch.NioSocketImpl$1.read(NioSocketImpl.java:803)
	at java.base/java.net.Socket$SocketInputStream.read(Socket.java:966)
	at java.base/sun.security.ssl.SSLSocketInputRecord.read(SSLSocketInputRecord.java:478)
	at java.base/sun.security.ssl.SSLSocketInputRecord.readHeader(SSLSocketInputRecord.java:472)
	at java.base/sun.security.ssl.SSLSocketInputRecord.bytesInCompletePacket(SSLSocketInputRecord.java:70)
	at java.base/sun.security.ssl.SSLSocketImpl.readApplicationRecord(SSLSocketImpl.java:1455)
	at java.base/sun.security.ssl.SSLSocketImpl$AppInputStream.read(SSLSocketImpl.java:1059)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:280)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:157)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:551)
	... 58 more


FAILED:  org.apache.solr.handler.TestSolrConfigHandlerCloud.test

Error Message:
java.lang.AssertionError: {
  "responseHeader":{
    "status":500,
    "QTime":30352},
  "errorMessages":["3 out of 5 the property overlay to be of version 8 within 30 seconds! Failed cores: [http://127.0.0.1:62593/g_wc/collection1_shard2_replica_n6/, http://127.0.0.1:62552/g_wc/collection1_shard1_replica_n1/, http://127.0.0.1:62599/g_wc/collection1_shard1_replica_n5/]\n"],
  "WARNING":"This response format is experimental.  It is likely to change in the future.",
  "error":{
    "metadata":[
      "error-class","org.apache.solr.common.SolrException",
      "root-error-class","org.apache.solr.common.SolrException"],
    "msg":"3 out of 5 the property overlay to be of version 8 within 30 seconds! Failed cores: [http://127.0.0.1:62593/g_wc/collection1_shard2_replica_n6/, http://127.0.0.1:62552/g_wc/collection1_shard1_replica_n1/, http://127.0.0.1:62599/g_wc/collection1_shard1_replica_n5/]",
    "trace":"org.apache.solr.common.SolrException: 3 out of 5 the property overlay to be of version 8 within 30 seconds! Failed cores: [http://127.0.0.1:62593/g_wc/collection1_shard2_replica_n6/, http://127.0.0.1:62552/g_wc/collection1_shard1_replica_n1/, http://127.0.0.1:62599/g_wc/collection1_shard1_replica_n5/]\n\tat org.apache.solr.handler.SolrConfigHandler.waitForAllReplicasState(SolrConfigHandler.java:812)\n\tat org.apache.solr.handler.SolrConfigHandler$Command.handleCommands(SolrConfigHandler.java:534)\n\tat org.apache.solr.handler.SolrConfigHandler$Command.handlePOST(SolrConfigHandler.java:366)\n\tat org.apache.solr.handler.SolrConfigHandler.handleRequestBody(SolrConfigHandler.java:137)\n\tat org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:214)\n\tat org.apache.solr.core.SolrCore.execute(SolrCore.java:2638)\n\tat org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:824)\n\tat org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:564)\n\tat org.apache.solr.servlet.SolrDispatchFilter.dispatch(SolrDispatchFilter.java:218)\n\tat org.apache.solr.servlet.SolrDispatchFilter.lambda$doFilter$0(SolrDispatchFilter.java:189)\n\tat org.apache.solr.servlet.ServletUtils.traceHttpRequestExecution2(ServletUtils.java:232)\n\tat org.apache.solr.servlet.ServletUtils.rateLimitRequest(ServletUtils.java:208)\n\tat org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:187)\n\tat org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:171)\n\tat org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)\n\tat org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)\n\tat org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:189)\n\tat org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)\n\tat org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)\n\tat org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)\n\tat org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)\n\tat org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1434)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)\n\tat org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)\n\tat org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)\n\tat org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1349)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)\n\tat org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)\n\tat org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)\n\tat org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:713)\n\tat org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)\n\tat org.eclipse.jetty.server.Server.handle(Server.java:516)\n\tat org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:400)\n\tat org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:645)\n\tat org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:392)\n\tat org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:277)\n\tat org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311)\n\tat org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:105)\n\tat org.eclipse.jetty.io.ChannelEndPoint$1.run(ChannelEndPoint.java:104)\n\tat org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)\n\tat org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n",
    "code":500}}
 expected null, but was:<[3 out of 5 the property overlay to be of version 8 within 30 seconds! Failed cores: [http://127.0.0.1:62593/g_wc/collection1_shard2_replica_n6/, http://127.0.0.1:62552/g_wc/collection1_shard1_replica_n1/, http://127.0.0.1:62599/g_wc/collection1_shard1_replica_n5/]
]>

Stack Trace:
java.lang.AssertionError: {
  "responseHeader":{
    "status":500,
    "QTime":30352},
  "errorMessages":["3 out of 5 the property overlay to be of version 8 within 30 seconds! Failed cores: [http://127.0.0.1:62593/g_wc/collection1_shard2_replica_n6/, http://127.0.0.1:62552/g_wc/collection1_shard1_replica_n1/, http://127.0.0.1:62599/g_wc/collection1_shard1_replica_n5/]\n"],
  "WARNING":"This response format is experimental.  It is likely to change in the future.",
  "error":{
    "metadata":[
      "error-class","org.apache.solr.common.SolrException",
      "root-error-class","org.apache.solr.common.SolrException"],
    "msg":"3 out of 5 the property overlay to be of version 8 within 30 seconds! Failed cores: [http://127.0.0.1:62593/g_wc/collection1_shard2_replica_n6/, http://127.0.0.1:62552/g_wc/collection1_shard1_replica_n1/, http://127.0.0.1:62599/g_wc/collection1_shard1_replica_n5/]",
    "trace":"org.apache.solr.common.SolrException: 3 out of 5 the property overlay to be of version 8 within 30 seconds! Failed cores: [http://127.0.0.1:62593/g_wc/collection1_shard2_replica_n6/, http://127.0.0.1:62552/g_wc/collection1_shard1_replica_n1/, http://127.0.0.1:62599/g_wc/collection1_shard1_replica_n5/]\n\tat org.apache.solr.handler.SolrConfigHandler.waitForAllReplicasState(SolrConfigHandler.java:812)\n\tat org.apache.solr.handler.SolrConfigHandler$Command.handleCommands(SolrConfigHandler.java:534)\n\tat org.apache.solr.handler.SolrConfigHandler$Command.handlePOST(SolrConfigHandler.java:366)\n\tat org.apache.solr.handler.SolrConfigHandler.handleRequestBody(SolrConfigHandler.java:137)\n\tat org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:214)\n\tat org.apache.solr.core.SolrCore.execute(SolrCore.java:2638)\n\tat org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:824)\n\tat org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:564)\n\tat org.apache.solr.servlet.SolrDispatchFilter.dispatch(SolrDispatchFilter.java:218)\n\tat org.apache.solr.servlet.SolrDispatchFilter.lambda$doFilter$0(SolrDispatchFilter.java:189)\n\tat org.apache.solr.servlet.ServletUtils.traceHttpRequestExecution2(ServletUtils.java:232)\n\tat org.apache.solr.servlet.ServletUtils.rateLimitRequest(ServletUtils.java:208)\n\tat org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:187)\n\tat org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:171)\n\tat org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)\n\tat org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)\n\tat org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:189)\n\tat org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)\n\tat org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)\n\tat org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)\n\tat org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)\n\tat org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1434)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)\n\tat org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)\n\tat org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)\n\tat org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1349)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)\n\tat org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)\n\tat org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)\n\tat org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:713)\n\tat org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)\n\tat org.eclipse.jetty.server.Server.handle(Server.java:516)\n\tat org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:400)\n\tat org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:645)\n\tat org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:392)\n\tat org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:277)\n\tat org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311)\n\tat org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:105)\n\tat org.eclipse.jetty.io.ChannelEndPoint$1.run(ChannelEndPoint.java:104)\n\tat org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)\n\tat org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)\n\tat java.base/java.lang.Thread.run(Thread.java:833)\n",
    "code":500}}
 expected null, but was:<[3 out of 5 the property overlay to be of version 8 within 30 seconds! Failed cores: [http://127.0.0.1:62593/g_wc/collection1_shard2_replica_n6/, http://127.0.0.1:62552/g_wc/collection1_shard1_replica_n1/, http://127.0.0.1:62599/g_wc/collection1_shard1_replica_n5/]
]>
	at __randomizedtesting.SeedInfo.seed([33FE595988D408C4:BBAA66832628653C]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.junit.Assert.failNotNull(Assert.java:756)
	at org.junit.Assert.assertNull(Assert.java:738)
	at org.apache.solr.core.TestSolrConfigHandler.runConfigCommand(TestSolrConfigHandler.java:221)
	at org.apache.solr.core.TestSolrConfigHandler.reqhandlertests(TestSolrConfigHandler.java:367)
	at org.apache.solr.handler.TestSolrConfigHandlerCloud.testReqHandlerAPIs(TestSolrConfigHandlerCloud.java:80)
	at org.apache.solr.handler.TestSolrConfigHandlerCloud.test(TestSolrConfigHandlerCloud.java:50)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:568)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1081)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1052)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:833)


FAILED:  org.apache.solr.core.TestLazyCores.testDontEvictUsedCore

Error Message:
java.lang.AssertionError

Stack Trace:
java.lang.AssertionError
	at __randomizedtesting.SeedInfo.seed([33FE595988D408C4:446D9880D3615040]:0)
	at org.junit.Assert.fail(Assert.java:87)
	at org.junit.Assert.assertTrue(Assert.java:42)
	at org.junit.Assert.assertTrue(Assert.java:53)
	at org.apache.solr.core.TestLazyCores.testDontEvictUsedCore(TestLazyCores.java:937)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:568)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:833)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
java.security.AccessControlException: access denied ("java.io.FilePermission" "/Users/jenkins/workspace/Solr-main-MacOSX/solr/core/build/resources/test/solr/filestore" "write")

Stack Trace:
java.security.AccessControlException: access denied ("java.io.FilePermission" "/Users/jenkins/workspace/Solr-main-MacOSX/solr/core/build/resources/test/solr/filestore" "write")
	at __randomizedtesting.SeedInfo.seed([33FE595988D408C4]:0)
	at java.base/java.security.AccessControlContext.checkPermission(AccessControlContext.java:485)
	at java.base/java.security.AccessController.checkPermission(AccessController.java:1068)
	at java.base/java.lang.SecurityManager.checkPermission(SecurityManager.java:416)
	at java.base/java.lang.SecurityManager.checkWrite(SecurityManager.java:847)
	at java.base/sun.nio.fs.UnixPath.checkWrite(UnixPath.java:787)
	at java.base/sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:389)
	at java.base/java.nio.file.Files.createDirectory(Files.java:700)
	at java.base/java.nio.file.Files.createAndCheckIsDirectory(Files.java:807)
	at java.base/java.nio.file.Files.createDirectories(Files.java:753)
	at org.apache.solr.filestore.DistribPackageStore.getPackageStoreDirPath(DistribPackageStore.java:584)
	at org.apache.solr.filestore.DistribPackageStore.ensurePackageStoreDir(DistribPackageStore.java:567)
	at org.apache.solr.filestore.DistribPackageStore.<init>(DistribPackageStore.java:83)
	at org.apache.solr.filestore.PackageStoreAPI.<init>(PackageStoreAPI.java:76)
	at org.apache.solr.core.CoreContainer.load(CoreContainer.java:725)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:184)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:156)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:162)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:112)
	at org.apache.solr.SolrTestCaseJ4.createCore(SolrTestCaseJ4.java:813)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:803)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:650)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:639)
	at org.apache.solr.cloud.AbstractZkTestCase.azt_beforeClass(AbstractZkTestCase.java:68)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:568)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:882)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:833)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 8 threads leaked from SUITE scope at org.apache.solr.rest.TestManagedResourceStorage: 
   1) Thread[id=28344, name=httpShardExecutor-17687-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@17.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$272/0x0000000800f2e880.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$235/0x0000000800ee25e8.run(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   2) Thread[id=28352, name=SUITE-TestManagedResourceStorage-seed#[33FE595988D408C4]-worker-EventThread, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3463)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3434)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at java.base@17.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:551)
   3) Thread[id=28353, name=zkConnectionManagerCallback-17694-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3463)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3434)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at java.base@17.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   4) Thread[id=28351, name=SUITE-TestManagedResourceStorage-seed#[33FE595988D408C4]-worker-SendThread(127.0.0.1:63837), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:214)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1380)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanAndNotifyState(ClientCnxn.java:1321)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1299)
   5) Thread[id=28356, name=zkCallback-17693-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@17.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   6) Thread[id=28355, name=OverseerCollectionConfigSetProcessor-72085774158397443-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:271)
        at app//org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:258)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   7) Thread[id=28347, name=h2sc-17689-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@17.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$272/0x0000000800f2e880.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$235/0x0000000800ee25e8.run(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   8) Thread[id=28354, name=OverseerStateUpdate-72085774158397443-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:294)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 8 threads leaked from SUITE scope at org.apache.solr.rest.TestManagedResourceStorage: 
   1) Thread[id=28344, name=httpShardExecutor-17687-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@17.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$272/0x0000000800f2e880.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$235/0x0000000800ee25e8.run(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   2) Thread[id=28352, name=SUITE-TestManagedResourceStorage-seed#[33FE595988D408C4]-worker-EventThread, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3463)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3434)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at java.base@17.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:551)
   3) Thread[id=28353, name=zkConnectionManagerCallback-17694-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3463)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3434)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at java.base@17.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   4) Thread[id=28351, name=SUITE-TestManagedResourceStorage-seed#[33FE595988D408C4]-worker-SendThread(127.0.0.1:63837), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:214)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1380)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanAndNotifyState(ClientCnxn.java:1321)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1299)
   5) Thread[id=28356, name=zkCallback-17693-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@17.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   6) Thread[id=28355, name=OverseerCollectionConfigSetProcessor-72085774158397443-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:271)
        at app//org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:258)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   7) Thread[id=28347, name=h2sc-17689-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@17.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$272/0x0000000800f2e880.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$235/0x0000000800ee25e8.run(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   8) Thread[id=28354, name=OverseerStateUpdate-72085774158397443-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:294)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
	at __randomizedtesting.SeedInfo.seed([33FE595988D408C4]:0)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=28344, name=httpShardExecutor-17687-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@17.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$272/0x0000000800f2e880.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$235/0x0000000800ee25e8.run(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   2) Thread[id=28353, name=zkConnectionManagerCallback-17694-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3463)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3434)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at java.base@17.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   3) Thread[id=28351, name=SUITE-TestManagedResourceStorage-seed#[33FE595988D408C4]-worker-SendThread(127.0.0.1:63837), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:362)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1197)
   4) Thread[id=28356, name=zkCallback-17693-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@17.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   5) Thread[id=28347, name=h2sc-17689-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@17.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$272/0x0000000800f2e880.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$235/0x0000000800ee25e8.run(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=28344, name=httpShardExecutor-17687-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@17.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$272/0x0000000800f2e880.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$235/0x0000000800ee25e8.run(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   2) Thread[id=28353, name=zkConnectionManagerCallback-17694-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3463)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3434)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at java.base@17.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   3) Thread[id=28351, name=SUITE-TestManagedResourceStorage-seed#[33FE595988D408C4]-worker-SendThread(127.0.0.1:63837), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:362)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1197)
   4) Thread[id=28356, name=zkCallback-17693-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@17.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   5) Thread[id=28347, name=h2sc-17689-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@17.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$272/0x0000000800f2e880.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$235/0x0000000800ee25e8.run(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
	at __randomizedtesting.SeedInfo.seed([33FE595988D408C4]:0)

[JENKINS] Solr-main-MacOSX (64bit/jdk-17.0.2) - Build # 664 - Failure!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/664/
Java: 64bit/jdk-17.0.2 -XX:+UseCompressedOops -XX:+UseG1GC

12 tests failed.
FAILED:  org.apache.solr.core.TestLazyCores.testDontEvictUsedCore

Error Message:
java.lang.AssertionError

Stack Trace:
java.lang.AssertionError
	at __randomizedtesting.SeedInfo.seed([15E7EC49180A1D8D:62742D9043BF4509]:0)
	at org.junit.Assert.fail(Assert.java:87)
	at org.junit.Assert.assertTrue(Assert.java:42)
	at org.junit.Assert.assertTrue(Assert.java:53)
	at org.apache.solr.core.TestLazyCores.testDontEvictUsedCore(TestLazyCores.java:937)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:568)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:833)


FAILED:  org.apache.solr.core.snapshots.TestSolrCloudSnapshots.testSnapshots

Error Message:
java.lang.AssertionError: expected:<COMPLETED> but was:<RUNNING>

Stack Trace:
java.lang.AssertionError: expected:<COMPLETED> but was:<RUNNING>
	at __randomizedtesting.SeedInfo.seed([15E7EC49180A1D8D:23AB68D66D531FC2]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.junit.Assert.failNotEquals(Assert.java:835)
	at org.junit.Assert.assertEquals(Assert.java:120)
	at org.junit.Assert.assertEquals(Assert.java:146)
	at org.apache.solr.core.snapshots.TestSolrCloudSnapshots.testSnapshots(TestSolrCloudSnapshots.java:204)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:568)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:833)


FAILED:  org.apache.solr.handler.TestSystemCollAutoCreate.testAutoCreate

Error Message:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:57326/mp/d: Underlying core creation failed while creating collection: control_collection

Stack Trace:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:57326/mp/d: Underlying core creation failed while creating collection: control_collection
	at __randomizedtesting.SeedInfo.seed([15E7EC49180A1D8D:2ABC5898AE74D4E0]:0)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:668)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:239)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:222)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:389)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:359)
	at app//org.apache.solr.client.solrj.impl.BaseCloudSolrClient.sendRequest(BaseCloudSolrClient.java:1134)
	at app//org.apache.solr.client.solrj.impl.BaseCloudSolrClient.requestWithRetryOnStaleState(BaseCloudSolrClient.java:898)
	at app//org.apache.solr.client.solrj.impl.BaseCloudSolrClient.request(BaseCloudSolrClient.java:832)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:226)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:243)
	at app//org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:339)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1079)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1052)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)


FAILED:  org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=2647, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@17.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:156)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:89)
        at app//org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:342)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:82)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getLines(AbstractAnalysisFactory.java:302)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getWordSet(AbstractAnalysisFactory.java:293)
        at app//org.apache.lucene.analysis.util.ElisionFilterFactory.inform(ElisionFilterFactory.java:73)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:723)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1341)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1194)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1179)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:615)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:177)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:107)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:244)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:314)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:241)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$607/0x000000080113fb58.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:46)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:239)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1686)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3134)
        at app//org.apache.solr.core.SolrCore$$Lambda$730/0x000000080129cf48.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2499)
        at app//org.apache.solr.cloud.ZkController$$Lambda$864/0x00000008012f7dc0.run(Unknown Source)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=2647, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@17.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:156)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:89)
        at app//org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:342)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:82)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getLines(AbstractAnalysisFactory.java:302)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getWordSet(AbstractAnalysisFactory.java:293)
        at app//org.apache.lucene.analysis.util.ElisionFilterFactory.inform(ElisionFilterFactory.java:73)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:723)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1341)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1194)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1179)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:615)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:177)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:107)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:244)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:314)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:241)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$607/0x000000080113fb58.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:46)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:239)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1686)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3134)
        at app//org.apache.solr.core.SolrCore$$Lambda$730/0x000000080129cf48.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2499)
        at app//org.apache.solr.cloud.ZkController$$Lambda$864/0x00000008012f7dc0.run(Unknown Source)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
	at __randomizedtesting.SeedInfo.seed([15E7EC49180A1D8D]:0)


FAILED:  org.apache.solr.schema.TestManagedSchemaAPI.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=14973, name=qtp143000575-14973, state=RUNNABLE, group=TGRP-TestManagedSchemaAPI], registration stack trace below.

Stack Trace:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=14973, name=qtp143000575-14973, state=RUNNABLE, group=TGRP-TestManagedSchemaAPI], registration stack trace below.
	at java.base@17.0.2/java.lang.Thread.getStackTrace(Thread.java:1610)
	at app//com.carrotsearch.randomizedtesting.RandomizedContext.closeAtEnd(RandomizedContext.java:157)
	at app//org.apache.lucene.util.LuceneTestCase.closeAfterSuite(LuceneTestCase.java:798)
	at app//org.apache.lucene.util.LuceneTestCase.wrapDirectory(LuceneTestCase.java:1504)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1368)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1350)
	at app//org.apache.solr.core.MockDirectoryFactory.create(MockDirectoryFactory.java:49)
	at app//org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:360)
	at app//org.apache.solr.core.SolrCore.initSnapshotMetaDataManager(SolrCore.java:582)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1026)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:961)
	at app//org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1423)
	at app//org.apache.solr.core.CoreContainer.create(CoreContainer.java:1297)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:345)
	at app//org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:423)
	at app//org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:197)
	at app//org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:214)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:866)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:830)
	at app//org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:542)
	at app//org.apache.solr.servlet.SolrDispatchFilter.dispatch(SolrDispatchFilter.java:218)
	at app//org.apache.solr.servlet.SolrDispatchFilter.lambda$doFilter$0(SolrDispatchFilter.java:189)
	at app//org.apache.solr.servlet.ServletUtils.traceHttpRequestExecution2(ServletUtils.java:232)
	at app//org.apache.solr.servlet.ServletUtils.rateLimitRequest(ServletUtils.java:208)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:187)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:171)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:189)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1434)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
	at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)
	at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1349)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
	at app//org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:713)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.server.Server.handle(Server.java:516)
	at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:400)
	at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:645)
	at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:392)
	at app//org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:352)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
	at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
Caused by: java.lang.AssertionError: Directory not closed: MockDirectoryWrapper(ByteBuffersDirectory@40428f92 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7308d060)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.lucene.util.CloseableDirectory.close(CloseableDirectory.java:43)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:788)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:785)
	at com.carrotsearch.randomizedtesting.RandomizedContext.closeResources(RandomizedContext.java:225)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.afterAlways(RandomizedRunner.java:801)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	... 1 more


FAILED:  org.apache.solr.schema.TestManagedSchemaAPI.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=14973, name=qtp143000575-14973, state=RUNNABLE, group=TGRP-TestManagedSchemaAPI], registration stack trace below.

Stack Trace:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=14973, name=qtp143000575-14973, state=RUNNABLE, group=TGRP-TestManagedSchemaAPI], registration stack trace below.
	at java.base@17.0.2/java.lang.Thread.getStackTrace(Thread.java:1610)
	at app//com.carrotsearch.randomizedtesting.RandomizedContext.closeAtEnd(RandomizedContext.java:157)
	at app//org.apache.lucene.util.LuceneTestCase.closeAfterSuite(LuceneTestCase.java:798)
	at app//org.apache.lucene.util.LuceneTestCase.wrapDirectory(LuceneTestCase.java:1504)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1368)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1350)
	at app//org.apache.solr.core.MockDirectoryFactory.create(MockDirectoryFactory.java:49)
	at app//org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:360)
	at app//org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:404)
	at app//org.apache.solr.core.SolrCore.initIndex(SolrCore.java:817)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1030)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:961)
	at app//org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1423)
	at app//org.apache.solr.core.CoreContainer.create(CoreContainer.java:1297)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:345)
	at app//org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:423)
	at app//org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:197)
	at app//org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:214)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:866)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:830)
	at app//org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:542)
	at app//org.apache.solr.servlet.SolrDispatchFilter.dispatch(SolrDispatchFilter.java:218)
	at app//org.apache.solr.servlet.SolrDispatchFilter.lambda$doFilter$0(SolrDispatchFilter.java:189)
	at app//org.apache.solr.servlet.ServletUtils.traceHttpRequestExecution2(ServletUtils.java:232)
	at app//org.apache.solr.servlet.ServletUtils.rateLimitRequest(ServletUtils.java:208)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:187)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:171)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:189)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1434)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
	at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)
	at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1349)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
	at app//org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:713)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.server.Server.handle(Server.java:516)
	at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:400)
	at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:645)
	at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:392)
	at app//org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:352)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
	at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
Caused by: java.lang.AssertionError: Directory not closed: MockDirectoryWrapper(ByteBuffersDirectory@62717a69 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@518bd780)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.lucene.util.CloseableDirectory.close(CloseableDirectory.java:43)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:788)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:785)
	at com.carrotsearch.randomizedtesting.RandomizedContext.closeResources(RandomizedContext.java:225)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.afterAlways(RandomizedRunner.java:801)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	... 1 more


FAILED:  org.apache.solr.schema.TestManagedSchemaAPI.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=14973, name=qtp143000575-14973, state=RUNNABLE, group=TGRP-TestManagedSchemaAPI], registration stack trace below.

Stack Trace:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=14973, name=qtp143000575-14973, state=RUNNABLE, group=TGRP-TestManagedSchemaAPI], registration stack trace below.
	at java.base@17.0.2/java.lang.Thread.getStackTrace(Thread.java:1610)
	at app//com.carrotsearch.randomizedtesting.RandomizedContext.closeAtEnd(RandomizedContext.java:157)
	at app//org.apache.lucene.util.LuceneTestCase.closeAfterSuite(LuceneTestCase.java:798)
	at app//org.apache.lucene.util.LuceneTestCase.wrapDirectory(LuceneTestCase.java:1504)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1368)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1350)
	at app//org.apache.solr.core.MockDirectoryFactory.create(MockDirectoryFactory.java:49)
	at app//org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:360)
	at app//org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:98)
	at app//org.apache.solr.core.SolrCore.initIndex(SolrCore.java:849)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1030)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:961)
	at app//org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1423)
	at app//org.apache.solr.core.CoreContainer.create(CoreContainer.java:1297)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:345)
	at app//org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:423)
	at app//org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:197)
	at app//org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:214)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:866)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:830)
	at app//org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:542)
	at app//org.apache.solr.servlet.SolrDispatchFilter.dispatch(SolrDispatchFilter.java:218)
	at app//org.apache.solr.servlet.SolrDispatchFilter.lambda$doFilter$0(SolrDispatchFilter.java:189)
	at app//org.apache.solr.servlet.ServletUtils.traceHttpRequestExecution2(ServletUtils.java:232)
	at app//org.apache.solr.servlet.ServletUtils.rateLimitRequest(ServletUtils.java:208)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:187)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:171)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:189)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1434)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
	at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)
	at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1349)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
	at app//org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:713)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.server.Server.handle(Server.java:516)
	at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:400)
	at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:645)
	at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:392)
	at app//org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:352)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
	at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
Caused by: java.lang.AssertionError: Directory not closed: MockDirectoryWrapper(ByteBuffersDirectory@45eb5ec3 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@44f8a019)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.lucene.util.CloseableDirectory.close(CloseableDirectory.java:43)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:788)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:785)
	at com.carrotsearch.randomizedtesting.RandomizedContext.closeResources(RandomizedContext.java:225)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.afterAlways(RandomizedRunner.java:801)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	... 1 more


FAILED:  org.apache.solr.schema.TestManagedSchemaAPI.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.schema.TestManagedSchemaAPI: 
   1) Thread[id=15069, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestManagedSchemaAPI]
        at java.base@17.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:156)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:89)
        at app//org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:306)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.lookupZKManagedSchemaPath(ManagedIndexSchemaFactory.java:113)
        at app//org.apache.solr.schema.ZkIndexSchemaReader.<init>(ZkIndexSchemaReader.java:51)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.inform(ManagedIndexSchemaFactory.java:478)
        at app//org.apache.solr.core.SolrResourceLoader.inform(SolrResourceLoader.java:688)
        at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1061)
        at app//org.apache.solr.core.SolrCore.reload(SolrCore.java:744)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1690)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3134)
        at app//org.apache.solr.core.SolrCore$$Lambda$733/0x0000000801238258.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2499)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1308/0x0000000801677d30.run(Unknown Source)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   2) Thread[id=15080, name=searcherExecutor-10108-thread-1-processing-127.0.0.1:64908_solr, state=WAITING, group=TGRP-TestManagedSchemaAPI]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
        at java.base@17.0.2/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:230)
        at app//org.apache.solr.core.SolrCore.lambda$new$2(SolrCore.java:1048)
        at app//org.apache.solr.core.SolrCore$$Lambda$710/0x0000000801222580.call(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.FutureTask.run(FutureTask.java:264)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$257/0x0000000800f8eab0.run(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.schema.TestManagedSchemaAPI: 
   1) Thread[id=15069, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestManagedSchemaAPI]
        at java.base@17.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:156)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:89)
        at app//org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:306)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.lookupZKManagedSchemaPath(ManagedIndexSchemaFactory.java:113)
        at app//org.apache.solr.schema.ZkIndexSchemaReader.<init>(ZkIndexSchemaReader.java:51)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.inform(ManagedIndexSchemaFactory.java:478)
        at app//org.apache.solr.core.SolrResourceLoader.inform(SolrResourceLoader.java:688)
        at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1061)
        at app//org.apache.solr.core.SolrCore.reload(SolrCore.java:744)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1690)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3134)
        at app//org.apache.solr.core.SolrCore$$Lambda$733/0x0000000801238258.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2499)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1308/0x0000000801677d30.run(Unknown Source)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   2) Thread[id=15080, name=searcherExecutor-10108-thread-1-processing-127.0.0.1:64908_solr, state=WAITING, group=TGRP-TestManagedSchemaAPI]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047)
        at java.base@17.0.2/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:230)
        at app//org.apache.solr.core.SolrCore.lambda$new$2(SolrCore.java:1048)
        at app//org.apache.solr.core.SolrCore$$Lambda$710/0x0000000801222580.call(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.FutureTask.run(FutureTask.java:264)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$257/0x0000000800f8eab0.run(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
	at __randomizedtesting.SeedInfo.seed([15E7EC49180A1D8D]:0)


FAILED:  org.apache.solr.search.TestRecoveryHdfs.testExistOldBufferLog

Error Message:
java.util.concurrent.TimeoutException: Timeout waiting for finish replay updates

Stack Trace:
java.util.concurrent.TimeoutException: Timeout waiting for finish replay updates
	at __randomizedtesting.SeedInfo.seed([15E7EC49180A1D8D:4BB7F11C96C58D04]:0)
	at org.apache.solr.util.TimeOut.waitFor(TimeOut.java:66)
	at org.apache.solr.search.TestRecoveryHdfs.testExistOldBufferLog(TestRecoveryHdfs.java:570)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:568)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:833)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
java.security.AccessControlException: access denied ("java.io.FilePermission" "/Users/jenkins/workspace/Solr-main-MacOSX/solr/core/build/resources/test/solr/filestore" "write")

Stack Trace:
java.security.AccessControlException: access denied ("java.io.FilePermission" "/Users/jenkins/workspace/Solr-main-MacOSX/solr/core/build/resources/test/solr/filestore" "write")
	at __randomizedtesting.SeedInfo.seed([15E7EC49180A1D8D]:0)
	at java.base/java.security.AccessControlContext.checkPermission(AccessControlContext.java:485)
	at java.base/java.security.AccessController.checkPermission(AccessController.java:1068)
	at java.base/java.lang.SecurityManager.checkPermission(SecurityManager.java:416)
	at java.base/java.lang.SecurityManager.checkWrite(SecurityManager.java:847)
	at java.base/sun.nio.fs.UnixPath.checkWrite(UnixPath.java:787)
	at java.base/sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:389)
	at java.base/java.nio.file.Files.createDirectory(Files.java:700)
	at java.base/java.nio.file.Files.createAndCheckIsDirectory(Files.java:807)
	at java.base/java.nio.file.Files.createDirectories(Files.java:753)
	at org.apache.solr.filestore.DistribPackageStore.getPackageStoreDirPath(DistribPackageStore.java:584)
	at org.apache.solr.filestore.DistribPackageStore.ensurePackageStoreDir(DistribPackageStore.java:567)
	at org.apache.solr.filestore.DistribPackageStore.<init>(DistribPackageStore.java:83)
	at org.apache.solr.filestore.PackageStoreAPI.<init>(PackageStoreAPI.java:76)
	at org.apache.solr.core.CoreContainer.load(CoreContainer.java:725)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:184)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:156)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:162)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:112)
	at org.apache.solr.SolrTestCaseJ4.createCore(SolrTestCaseJ4.java:813)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:803)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:650)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:639)
	at org.apache.solr.cloud.AbstractZkTestCase.azt_beforeClass(AbstractZkTestCase.java:68)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:568)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:882)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:833)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 8 threads leaked from SUITE scope at org.apache.solr.rest.TestManagedResourceStorage: 
   1) Thread[id=23848, name=zkCallback-15530-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@17.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   2) Thread[id=23846, name=OverseerStateUpdate-72082843469021187-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:294)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   3) Thread[id=23836, name=httpShardExecutor-15524-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@17.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$272/0x0000000800f2eec8.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$235/0x0000000800ee2a30.run(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   4) Thread[id=23845, name=zkConnectionManagerCallback-15531-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3463)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3434)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at java.base@17.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   5) Thread[id=23847, name=OverseerCollectionConfigSetProcessor-72082843469021187-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:271)
        at app//org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:258)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   6) Thread[id=23843, name=SUITE-TestManagedResourceStorage-seed#[15E7EC49180A1D8D]-worker-SendThread(127.0.0.1:62110), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:214)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1380)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanAndNotifyState(ClientCnxn.java:1321)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1299)
   7) Thread[id=23844, name=SUITE-TestManagedResourceStorage-seed#[15E7EC49180A1D8D]-worker-EventThread, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3463)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3434)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at java.base@17.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:551)
   8) Thread[id=23839, name=h2sc-15526-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@17.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$272/0x0000000800f2eec8.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$235/0x0000000800ee2a30.run(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 8 threads leaked from SUITE scope at org.apache.solr.rest.TestManagedResourceStorage: 
   1) Thread[id=23848, name=zkCallback-15530-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@17.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   2) Thread[id=23846, name=OverseerStateUpdate-72082843469021187-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:294)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   3) Thread[id=23836, name=httpShardExecutor-15524-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@17.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$272/0x0000000800f2eec8.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$235/0x0000000800ee2a30.run(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   4) Thread[id=23845, name=zkConnectionManagerCallback-15531-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3463)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3434)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at java.base@17.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   5) Thread[id=23847, name=OverseerCollectionConfigSetProcessor-72082843469021187-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:271)
        at app//org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:258)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   6) Thread[id=23843, name=SUITE-TestManagedResourceStorage-seed#[15E7EC49180A1D8D]-worker-SendThread(127.0.0.1:62110), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:214)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1380)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanAndNotifyState(ClientCnxn.java:1321)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1299)
   7) Thread[id=23844, name=SUITE-TestManagedResourceStorage-seed#[15E7EC49180A1D8D]-worker-EventThread, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3463)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3434)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at java.base@17.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:551)
   8) Thread[id=23839, name=h2sc-15526-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@17.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$272/0x0000000800f2eec8.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$235/0x0000000800ee2a30.run(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
	at __randomizedtesting.SeedInfo.seed([15E7EC49180A1D8D]:0)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=23848, name=zkCallback-15530-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@17.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   2) Thread[id=23836, name=httpShardExecutor-15524-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@17.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$272/0x0000000800f2eec8.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$235/0x0000000800ee2a30.run(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   3) Thread[id=23845, name=zkConnectionManagerCallback-15531-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3463)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3434)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at java.base@17.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   4) Thread[id=23843, name=SUITE-TestManagedResourceStorage-seed#[15E7EC49180A1D8D]-worker-SendThread(127.0.0.1:62110), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1136)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1200)
   5) Thread[id=23839, name=h2sc-15526-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@17.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$272/0x0000000800f2eec8.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$235/0x0000000800ee2a30.run(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=23848, name=zkCallback-15530-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@17.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   2) Thread[id=23836, name=httpShardExecutor-15524-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@17.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$272/0x0000000800f2eec8.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$235/0x0000000800ee2a30.run(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   3) Thread[id=23845, name=zkConnectionManagerCallback-15531-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3463)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3434)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at java.base@17.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   4) Thread[id=23843, name=SUITE-TestManagedResourceStorage-seed#[15E7EC49180A1D8D]-worker-SendThread(127.0.0.1:62110), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1136)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1200)
   5) Thread[id=23839, name=h2sc-15526-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@17.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$272/0x0000000800f2eec8.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$235/0x0000000800ee2a30.run(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
	at __randomizedtesting.SeedInfo.seed([15E7EC49180A1D8D]:0)

[JENKINS] Solr-main-MacOSX (64bit/jdk-16.0.2) - Build # 663 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/663/
Java: 64bit/jdk-16.0.2 -XX:-UseCompressedOops -XX:+UseShenandoahGC

3 tests failed.
FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
java.security.AccessControlException: access denied ("java.io.FilePermission" "/Users/jenkins/workspace/Solr-main-MacOSX/solr/core/build/resources/test/solr/filestore" "write")

Stack Trace:
java.security.AccessControlException: access denied ("java.io.FilePermission" "/Users/jenkins/workspace/Solr-main-MacOSX/solr/core/build/resources/test/solr/filestore" "write")
	at __randomizedtesting.SeedInfo.seed([679835843077347E]:0)
	at java.base/java.security.AccessControlContext.checkPermission(AccessControlContext.java:472)
	at java.base/java.security.AccessController.checkPermission(AccessController.java:1036)
	at java.base/java.lang.SecurityManager.checkPermission(SecurityManager.java:408)
	at java.base/java.lang.SecurityManager.checkWrite(SecurityManager.java:838)
	at java.base/sun.nio.fs.UnixPath.checkWrite(UnixPath.java:816)
	at java.base/sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:388)
	at java.base/java.nio.file.Files.createDirectory(Files.java:694)
	at java.base/java.nio.file.Files.createAndCheckIsDirectory(Files.java:801)
	at java.base/java.nio.file.Files.createDirectories(Files.java:747)
	at org.apache.solr.filestore.DistribPackageStore.getPackageStoreDirPath(DistribPackageStore.java:584)
	at org.apache.solr.filestore.DistribPackageStore.ensurePackageStoreDir(DistribPackageStore.java:567)
	at org.apache.solr.filestore.DistribPackageStore.<init>(DistribPackageStore.java:83)
	at org.apache.solr.filestore.PackageStoreAPI.<init>(PackageStoreAPI.java:76)
	at org.apache.solr.core.CoreContainer.load(CoreContainer.java:725)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:184)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:156)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:162)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:112)
	at org.apache.solr.SolrTestCaseJ4.createCore(SolrTestCaseJ4.java:813)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:803)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:650)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:639)
	at org.apache.solr.cloud.AbstractZkTestCase.azt_beforeClass(AbstractZkTestCase.java:68)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:78)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:567)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:882)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:831)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 8 threads leaked from SUITE scope at org.apache.solr.rest.TestManagedResourceStorage: 
   1) Thread[id=27032, name=SUITE-TestManagedResourceStorage-seed#[679835843077347E]-worker-SendThread(127.0.0.1:49758), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:214)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1380)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanAndNotifyState(ClientCnxn.java:1321)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1299)
   2) Thread[id=27036, name=OverseerCollectionConfigSetProcessor-72082089015902211-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:271)
        at app//org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:258)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   3) Thread[id=27025, name=httpShardExecutor-17216-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$271/0x0000000800f21df8.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$234/0x0000000800ed5b30.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   4) Thread[id=27037, name=zkCallback-17222-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   5) Thread[id=27033, name=SUITE-TestManagedResourceStorage-seed#[679835843077347E]-worker-EventThread, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:551)
   6) Thread[id=27034, name=zkConnectionManagerCallback-17223-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   7) Thread[id=27028, name=h2sc-17218-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$271/0x0000000800f21df8.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$234/0x0000000800ed5b30.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   8) Thread[id=27035, name=OverseerStateUpdate-72082089015902211-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:294)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 8 threads leaked from SUITE scope at org.apache.solr.rest.TestManagedResourceStorage: 
   1) Thread[id=27032, name=SUITE-TestManagedResourceStorage-seed#[679835843077347E]-worker-SendThread(127.0.0.1:49758), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:214)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1380)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanAndNotifyState(ClientCnxn.java:1321)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1299)
   2) Thread[id=27036, name=OverseerCollectionConfigSetProcessor-72082089015902211-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:271)
        at app//org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:258)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   3) Thread[id=27025, name=httpShardExecutor-17216-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$271/0x0000000800f21df8.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$234/0x0000000800ed5b30.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   4) Thread[id=27037, name=zkCallback-17222-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   5) Thread[id=27033, name=SUITE-TestManagedResourceStorage-seed#[679835843077347E]-worker-EventThread, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:551)
   6) Thread[id=27034, name=zkConnectionManagerCallback-17223-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   7) Thread[id=27028, name=h2sc-17218-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$271/0x0000000800f21df8.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$234/0x0000000800ed5b30.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   8) Thread[id=27035, name=OverseerStateUpdate-72082089015902211-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:294)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
	at __randomizedtesting.SeedInfo.seed([679835843077347E]:0)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=27032, name=SUITE-TestManagedResourceStorage-seed#[679835843077347E]-worker-SendThread(127.0.0.1:49758), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:362)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1197)
   2) Thread[id=27025, name=httpShardExecutor-17216-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$271/0x0000000800f21df8.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$234/0x0000000800ed5b30.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   3) Thread[id=27037, name=zkCallback-17222-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   4) Thread[id=27034, name=zkConnectionManagerCallback-17223-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   5) Thread[id=27028, name=h2sc-17218-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$271/0x0000000800f21df8.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$234/0x0000000800ed5b30.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=27032, name=SUITE-TestManagedResourceStorage-seed#[679835843077347E]-worker-SendThread(127.0.0.1:49758), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:362)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1197)
   2) Thread[id=27025, name=httpShardExecutor-17216-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$271/0x0000000800f21df8.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$234/0x0000000800ed5b30.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   3) Thread[id=27037, name=zkCallback-17222-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   4) Thread[id=27034, name=zkConnectionManagerCallback-17223-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   5) Thread[id=27028, name=h2sc-17218-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$271/0x0000000800f21df8.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$234/0x0000000800ed5b30.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
	at __randomizedtesting.SeedInfo.seed([679835843077347E]:0)

[JENKINS] Solr-main-MacOSX (64bit/jdk-11.0.13) - Build # 662 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/662/
Java: 64bit/jdk-11.0.13 -XX:+UseCompressedOops -XX:+UseConcMarkSweepGC

5 tests failed.
FAILED:  org.apache.solr.cloud.DocValuesNotIndexedTest.testGroupingDVOnlySortLast

Error Message:
java.lang.AssertionError: Unexpected number of elements in the group for 'intGSL' size: '8' GroupValue: 'null' rsp: {responseHeader={zkConnected=true, status=0, QTime=31, params={q=*:*, group.sort=id asc, _stateVer_=dv_coll:4, group.limit=100, rows=100, wt=javabin, version=2, group.field=intGSL, group=true}}, grouped={intGSL={matches=48, groups=[{groupValue=null, doclist={numFound=8,numFoundExact=true,start=0,maxScore=1.0,docs=[SolrDocument{id=10000, _version_=1725140713075638272, _root_=10000}, SolrDocument{id=10005, _version_=1725140713077735424, _root_=10005}, SolrDocument{id=10010, _version_=1725140713080881152, _root_=10010}, SolrDocument{id=10015, _version_=1725140713082978305, _root_=10015}, SolrDocument{id=10020, _version_=1725140713082978306, _root_=10020}, SolrDocument{id=10025, _version_=1725140713080881154, _root_=10025}, SolrDocument{id=10035, _version_=1725140713080881158, _root_=10035}, SolrDocument{id=10045, _version_=1725140713080881160, _root_=10045}]}}, {groupValue=1002066038, doclist={numFound=7,numFoundExact=true,start=0,maxScore=1.0,docs=[SolrDocument{id=0, intGSL=1002066038, longGSL=267061401255881305, doubleGSL=5465.073229265279, floatGSL=8639.165, dateGSL=Thu Dec 08 09:13:57 XJT 52533070, stringGSL=base_string_998753__00001682, boolGSL=true, sortableGSL=base_string_551223__00007129, _version_=1725140713074589696, _root_=0}, SolrDocument{id=1, intGSL=1002066038, longGSL=267061401255881305, doubleGSL=5465.073229265279, floatGSL=8639.165, dateGSL=Thu Dec 08 09:13:57 XJT 52533070, stringGSL=base_string_998753__00001682, boolGSL=false, sortableGSL=base_string_551223__00007129, _version_=1725140713075638272, _root_=1}, SolrDocument{id=2, intGSL=1002066038, longGSL=267061401255881305, doubleGSL=5465.073229265279, floatGSL=8639.165, dateGSL=Thu Dec 08 09:13:57 XJT 52533070, stringGSL=base_string_998753__00001682, boolGSL=true, sortableGSL=base_string_551223__00007129, _version_=1725140713075638272, _root_=2}, SolrDocument{id=3, intGSL=1002066038, longGSL=267061401255881305, doubleGSL=5465.073229265279, floatGSL=8639.165, dateGSL=Thu Dec 08 09:13:57 XJT 52533070, stringGSL=base_string_998753__00001682, boolGSL=false, sortableGSL=base_string_551223__00007129, _version_=1725140713077735424, _root_=3}, SolrDocument{id=4, intGSL=1002066038, longGSL=267061401255881305, doubleGSL=5465.073229265279, floatGSL=8639.165, dateGSL=Thu Dec 08 09:13:57 XJT 52533070, stringGSL=base_string_998753__00001682, boolGSL=true, sortableGSL=base_string_551223__00007129, _version_=1725140713076686848, _root_=4}, SolrDocument{id=5, intGSL=1002066038, longGSL=267061401255881305, doubleGSL=5465.073229265279, floatGSL=8639.165, dateGSL=Thu Dec 08 09:13:57 XJT 52533070, stringGSL=base_string_998753__00001682, boolGSL=false, sortableGSL=base_string_551223__00007129, _version_=1725140713079832576, _root_=5}, SolrDocument{id=6, intGSL=1002066038, longGSL=267061401255881305, doubleGSL=5465.073229265279, floatGSL=8639.165, dateGSL=Thu Dec 08 09:13:57 XJT 52533070, stringGSL=base_string_998753__00001682, boolGSL=true, sortableGSL=base_string_551223__00007129, _version_=1725140713078784000, _root_=6}]}}, {groupValue=1002066475, doclist={numFound=7,numFoundExact=true,start=0,maxScore=1.0,docs=[SolrDocument{id=10, intGSL=1002066475, longGSL=267061401255889156, doubleGSL=11119.07322926528, floatGSL=10295.165, dateGSL=Thu Dec 08 09:13:58 XJT 52533070, stringGSL=base_string_998753__00011209, boolGSL=true, sortableGSL=base_string_551223__00007785, _version_=1725140713077735425, _root_=10}, SolrDocument{id=11, intGSL=1002066475, longGSL=267061401255889156, doubleGSL=11119.07322926528, floatGSL=10295.165, dateGSL=Thu Dec 08 09:13:58 XJT 52533070, stringGSL=base_string_998753__00011209, boolGSL=false, sortableGSL=base_string_551223__00007785, _version_=1725140713077735426, _root_=11}, SolrDocument{id=12, intGSL=1002066475, longGSL=267061401255889156, doubleGSL=11119.07322926528, floatGSL=10295.165, dateGSL=Thu Dec 08 09:13:58 XJT 52533070, stringGSL=base_string_998753__00011209, boolGSL=true, sortableGSL=base_string_551223__00007785, _version_=1725140713077735425, _root_=12}, SolrDocument{id=13, intGSL=1002066475, longGSL=267061401255889156, doubleGSL=11119.07322926528, floatGSL=10295.165, dateGSL=Thu Dec 08 09:13:58 XJT 52533070, stringGSL=base_string_998753__00011209, boolGSL=false, sortableGSL=base_string_551223__00007785, _version_=1725140713079832576, _root_=13}, SolrDocument{id=7, intGSL=1002066475, longGSL=267061401255889156, doubleGSL=11119.07322926528, floatGSL=10295.165, dateGSL=Thu Dec 08 09:13:58 XJT 52533070, stringGSL=base_string_998753__00011209, boolGSL=false, sortableGSL=base_string_551223__00007785, _version_=1725140713078784001, _root_=7}, SolrDocument{id=8, intGSL=1002066475, longGSL=267061401255889156, doubleGSL=11119.07322926528, floatGSL=10295.165, dateGSL=Thu Dec 08 09:13:58 XJT 52533070, stringGSL=base_string_998753__00011209, boolGSL=true, sortableGSL=base_string_551223__00007785, _version_=1725140713077735424, _root_=8}, SolrDocument{id=9, intGSL=1002066475, longGSL=267061401255889156, doubleGSL=11119.07322926528, floatGSL=10295.165, dateGSL=Thu Dec 08 09:13:58 XJT 52533070, stringGSL=base_string_998753__00011209, boolGSL=false, sortableGSL=base_string_551223__00007785, _version_=1725140713079832576, _root_=9}]}}, {groupValue=1002072701, doclist={numFound=5,numFoundExact=true,start=0,maxScore=1.0,docs=[SolrDocument{id=14, intGSL=1002072701, longGSL=267061401255894648, doubleGSL=12294.07322926528, floatGSL=14984.165, dateGSL=Thu Dec 08 09:14:00 XJT 52533070, stringGSL=base_string_998753__00017372, boolGSL=true, sortableGSL=base_string_551223__00009830, _version_=1725140713082978304, _root_=14}, SolrDocument{id=15, intGSL=1002072701, longGSL=267061401255894648, doubleGSL=12294.07322926528, floatGSL=14984.165, dateGSL=Thu Dec 08 09:14:00 XJT 52533070, stringGSL=base_string_998753__00017372, boolGSL=false, sortableGSL=base_string_551223__00009830, _version_=1725140713077735426, _root_=15}, SolrDocument{id=16, intGSL=1002072701, longGSL=267061401255894648, doubleGSL=12294.07322926528, floatGSL=14984.165, dateGSL=Thu Dec 08 09:14:00 XJT 52533070, stringGSL=base_string_998753__00017372, boolGSL=true, sortableGSL=base_string_551223__00009830, _version_=1725140713078784000, _root_=16}, SolrDocument{id=18, intGSL=1002072701, longGSL=267061401255894648, doubleGSL=12294.07322926528, floatGSL=14984.165, dateGSL=Thu Dec 08 09:14:00 XJT 52533070, stringGSL=base_string_998753__00017372, boolGSL=true, sortableGSL=base_string_551223__00009830, _version_=1725140713080881153, _root_=18}, SolrDocument{id=20, intGSL=1002072701, longGSL=267061401255894648, doubleGSL=12294.07322926528, floatGSL=14984.165, dateGSL=Thu Dec 08 09:14:00 XJT 52533070, stringGSL=base_string_998753__00017372, boolGSL=true, sortableGSL=base_string_551223__00009830, _version_=1725140713079832576, _root_=20}]}}, {groupValue=1002075300, doclist={numFound=4,numFoundExact=true,start=0,maxScore=1.0,docs=[SolrDocument{id=22, intGSL=1002075300, longGSL=267061401255898482, doubleGSL=16273.07322926528, floatGSL=24685.164, dateGSL=Thu Dec 08 09:14:10 XJT 52533070, stringGSL=base_string_998753__00020157, boolGSL=true, sortableGSL=base_string_551223__00013613, _version_=1725140713079832577, _root_=22}, SolrDocument{id=24, intGSL=1002075300, longGSL=267061401255898482, doubleGSL=16273.07322926528, floatGSL=24685.164, dateGSL=Thu Dec 08 09:14:10 XJT 52533070, stringGSL=base_string_998753__00020157, boolGSL=true, sortableGSL=base_string_551223__00013613, _version_=1725140713082978307, _root_=24}, SolrDocument{id=26, intGSL=1002075300, longGSL=267061401255898482, doubleGSL=16273.07322926528, floatGSL=24685.164, dateGSL=Thu Dec 08 09:14:10 XJT 52533070, stringGSL=base_string_998753__00020157, boolGSL=true, sortableGSL=base_string_551223__00013613, _version_=1725140713080881155, _root_=26}, SolrDocument{id=27, intGSL=1002075300, longGSL=267061401255898482, doubleGSL=16273.07322926528, floatGSL=24685.164, dateGSL=Thu Dec 08 09:14:10 XJT 52533070, stringGSL=base_string_998753__00020157, boolGSL=false, sortableGSL=base_string_551223__00013613, _version_=1725140713082978308, _root_=27}]}}, {groupValue=1002083995, doclist={numFound=5,numFoundExact=true,start=0,maxScore=1.0,docs=[SolrDocument{id=28, intGSL=1002083995, longGSL=267061401255901499, doubleGSL=16812.07322926528, floatGSL=31805.164, dateGSL=Thu Dec 08 09:14:11 XJT 52533070, stringGSL=base_string_998753__00023957, boolGSL=true, sortableGSL=base_string_551223__00021286, _version_=1725140713082978309, _root_=28}, SolrDocument{id=30, intGSL=1002083995, longGSL=267061401255901499, doubleGSL=16812.07322926528, floatGSL=31805.164, dateGSL=Thu Dec 08 09:14:11 XJT 52533070, stringGSL=base_string_998753__00023957, boolGSL=true, sortableGSL=base_string_551223__00021286, _version_=1725140713080881156, _root_=30}, SolrDocument{id=31, intGSL=1002083995, longGSL=267061401255901499, doubleGSL=16812.07322926528, floatGSL=31805.164, dateGSL=Thu Dec 08 09:14:11 XJT 52533070, stringGSL=base_string_998753__00023957, boolGSL=false, sortableGSL=base_string_551223__00021286, _version_=1725140713080881157, _root_=31}, SolrDocument{id=32, intGSL=1002083995, longGSL=267061401255901499, doubleGSL=16812.07322926528, floatGSL=31805.164, dateGSL=Thu Dec 08 09:14:11 XJT 52533070, stringGSL=base_string_998753__00023957, boolGSL=true, sortableGSL=base_string_551223__00021286, _version_=1725140713082978310, _root_=32}, SolrDocument{id=34, intGSL=1002083995, longGSL=267061401255901499, doubleGSL=16812.07322926528, floatGSL=31805.164, dateGSL=Thu Dec 08 09:14:11 XJT 52533070, stringGSL=base_string_998753__00023957, boolGSL=true, sortableGSL=base_string_551223__00021286, _version_=1725140713079832578, _root_=34}]}}, {groupValue=1002087340, doclist={numFound=6,numFoundExact=true,start=0,maxScore=1.0,docs=[SolrDocument{id=35, intGSL=1002087340, longGSL=267061401255905558, doubleGSL=17899.07322926528, floatGSL=32053.164, dateGSL=Thu Dec 08 09:14:18 XJT 52533070, stringGSL=base_string_998753__00033035, boolGSL=false, sortableGSL=base_string_551223__00024779, _version_=1725140713079832579, _root_=35}, SolrDocument{id=36, intGSL=1002087340, longGSL=267061401255905558, doubleGSL=17899.07322926528, floatGSL=32053.164, dateGSL=Thu Dec 08 09:14:18 XJT 52533070, stringGSL=base_string_998753__00033035, boolGSL=true, sortableGSL=base_string_551223__00024779, _version_=1725140713079832580, _root_=36}, SolrDocument{id=38, intGSL=1002087340, longGSL=267061401255905558, doubleGSL=17899.07322926528, floatGSL=32053.164, dateGSL=Thu Dec 08 09:14:18 XJT 52533070, stringGSL=base_string_998753__00033035, boolGSL=true, sortableGSL=base_string_551223__00024779, _version_=1725140713082978311, _root_=38}, SolrDocument{id=39, intGSL=1002087340, longGSL=267061401255905558, doubleGSL=17899.07322926528, floatGSL=32053.164, dateGSL=Thu Dec 08 09:14:18 XJT 52533070, stringGSL=base_string_998753__00033035, boolGSL=false, sortableGSL=base_string_551223__00024779, _version_=1725140713080881159, _root_=39}, SolrDocument{id=40, intGSL=1002087340, longGSL=267061401255905558, doubleGSL=17899.07322926528, floatGSL=32053.164, dateGSL=Thu Dec 08 09:14:18 XJT 52533070, stringGSL=base_string_998753__00033035, boolGSL=true, sortableGSL=base_string_551223__00024779, _version_=1725140713082978312, _root_=40}, SolrDocument{id=41, intGSL=1002087340, longGSL=267061401255905558, doubleGSL=17899.07322926528, floatGSL=32053.164, dateGSL=Thu Dec 08 09:14:18 XJT 52533070, stringGSL=base_string_998753__00033035, boolGSL=false, sortableGSL=base_string_551223__00024779, _version_=1725140713079832581, _root_=41}]}}, {groupValue=1002094820, doclist={numFound=6,numFoundExact=true,start=0,maxScore=1.0,docs=[SolrDocument{id=42, intGSL=1002094820, longGSL=267061401255910036, doubleGSL=20223.07322926528, floatGSL=36366.164, dateGSL=Thu Dec 08 09:14:24 XJT 52533070, stringGSL=base_string_998753__00035431, boolGSL=true, sortableGSL=base_string_551223__00028733, _version_=1725140713082978313, _root_=42}, SolrDocument{id=43, intGSL=1002094820, longGSL=267061401255910036, doubleGSL=20223.07322926528, floatGSL=36366.164, dateGSL=Thu Dec 08 09:14:24 XJT 52533070, stringGSL=base_string_998753__00035431, boolGSL=false, sortableGSL=base_string_551223__00028733, _version_=1725140713079832582, _root_=43}, SolrDocument{id=44, intGSL=1002094820, longGSL=267061401255910036, doubleGSL=20223.07322926528, floatGSL=36366.164, dateGSL=Thu Dec 08 09:14:24 XJT 52533070, stringGSL=base_string_998753__00035431, boolGSL=true, sortableGSL=base_string_551223__00028733, _version_=1725140713079832583, _root_=44}, SolrDocument{id=46, intGSL=1002094820, longGSL=267061401255910036, doubleGSL=20223.07322926528, floatGSL=36366.164, dateGSL=Thu Dec 08 09:14:24 XJT 52533070, stringGSL=base_string_998753__00035431, boolGSL=true, sortableGSL=base_string_551223__00028733, _version_=1725140713079832584, _root_=46}, SolrDocument{id=47, intGSL=1002094820, longGSL=267061401255910036, doubleGSL=20223.07322926528, floatGSL=36366.164, dateGSL=Thu Dec 08 09:14:24 XJT 52533070, stringGSL=base_string_998753__00035431, boolGSL=false, sortableGSL=base_string_551223__00028733, _version_=1725140713079832585, _root_=47}, SolrDocument{id=48, intGSL=1002094820, longGSL=267061401255910036, doubleGSL=20223.07322926528, floatGSL=36366.164, dateGSL=Thu Dec 08 09:14:24 XJT 52533070, stringGSL=base_string_998753__00035431, boolGSL=true, sortableGSL=base_string_551223__00028733, _version_=1725140713082978314, _root_=48}]}}]}}}

Stack Trace:
java.lang.AssertionError: Unexpected number of elements in the group for 'intGSL' size: '8' GroupValue: 'null' rsp: {responseHeader={zkConnected=true, status=0, QTime=31, params={q=*:*, group.sort=id asc, _stateVer_=dv_coll:4, group.limit=100, rows=100, wt=javabin, version=2, group.field=intGSL, group=true}}, grouped={intGSL={matches=48, groups=[{groupValue=null, doclist={numFound=8,numFoundExact=true,start=0,maxScore=1.0,docs=[SolrDocument{id=10000, _version_=1725140713075638272, _root_=10000}, SolrDocument{id=10005, _version_=1725140713077735424, _root_=10005}, SolrDocument{id=10010, _version_=1725140713080881152, _root_=10010}, SolrDocument{id=10015, _version_=1725140713082978305, _root_=10015}, SolrDocument{id=10020, _version_=1725140713082978306, _root_=10020}, SolrDocument{id=10025, _version_=1725140713080881154, _root_=10025}, SolrDocument{id=10035, _version_=1725140713080881158, _root_=10035}, SolrDocument{id=10045, _version_=1725140713080881160, _root_=10045}]}}, {groupValue=1002066038, doclist={numFound=7,numFoundExact=true,start=0,maxScore=1.0,docs=[SolrDocument{id=0, intGSL=1002066038, longGSL=267061401255881305, doubleGSL=5465.073229265279, floatGSL=8639.165, dateGSL=Thu Dec 08 09:13:57 XJT 52533070, stringGSL=base_string_998753__00001682, boolGSL=true, sortableGSL=base_string_551223__00007129, _version_=1725140713074589696, _root_=0}, SolrDocument{id=1, intGSL=1002066038, longGSL=267061401255881305, doubleGSL=5465.073229265279, floatGSL=8639.165, dateGSL=Thu Dec 08 09:13:57 XJT 52533070, stringGSL=base_string_998753__00001682, boolGSL=false, sortableGSL=base_string_551223__00007129, _version_=1725140713075638272, _root_=1}, SolrDocument{id=2, intGSL=1002066038, longGSL=267061401255881305, doubleGSL=5465.073229265279, floatGSL=8639.165, dateGSL=Thu Dec 08 09:13:57 XJT 52533070, stringGSL=base_string_998753__00001682, boolGSL=true, sortableGSL=base_string_551223__00007129, _version_=1725140713075638272, _root_=2}, SolrDocument{id=3, intGSL=1002066038, longGSL=267061401255881305, doubleGSL=5465.073229265279, floatGSL=8639.165, dateGSL=Thu Dec 08 09:13:57 XJT 52533070, stringGSL=base_string_998753__00001682, boolGSL=false, sortableGSL=base_string_551223__00007129, _version_=1725140713077735424, _root_=3}, SolrDocument{id=4, intGSL=1002066038, longGSL=267061401255881305, doubleGSL=5465.073229265279, floatGSL=8639.165, dateGSL=Thu Dec 08 09:13:57 XJT 52533070, stringGSL=base_string_998753__00001682, boolGSL=true, sortableGSL=base_string_551223__00007129, _version_=1725140713076686848, _root_=4}, SolrDocument{id=5, intGSL=1002066038, longGSL=267061401255881305, doubleGSL=5465.073229265279, floatGSL=8639.165, dateGSL=Thu Dec 08 09:13:57 XJT 52533070, stringGSL=base_string_998753__00001682, boolGSL=false, sortableGSL=base_string_551223__00007129, _version_=1725140713079832576, _root_=5}, SolrDocument{id=6, intGSL=1002066038, longGSL=267061401255881305, doubleGSL=5465.073229265279, floatGSL=8639.165, dateGSL=Thu Dec 08 09:13:57 XJT 52533070, stringGSL=base_string_998753__00001682, boolGSL=true, sortableGSL=base_string_551223__00007129, _version_=1725140713078784000, _root_=6}]}}, {groupValue=1002066475, doclist={numFound=7,numFoundExact=true,start=0,maxScore=1.0,docs=[SolrDocument{id=10, intGSL=1002066475, longGSL=267061401255889156, doubleGSL=11119.07322926528, floatGSL=10295.165, dateGSL=Thu Dec 08 09:13:58 XJT 52533070, stringGSL=base_string_998753__00011209, boolGSL=true, sortableGSL=base_string_551223__00007785, _version_=1725140713077735425, _root_=10}, SolrDocument{id=11, intGSL=1002066475, longGSL=267061401255889156, doubleGSL=11119.07322926528, floatGSL=10295.165, dateGSL=Thu Dec 08 09:13:58 XJT 52533070, stringGSL=base_string_998753__00011209, boolGSL=false, sortableGSL=base_string_551223__00007785, _version_=1725140713077735426, _root_=11}, SolrDocument{id=12, intGSL=1002066475, longGSL=267061401255889156, doubleGSL=11119.07322926528, floatGSL=10295.165, dateGSL=Thu Dec 08 09:13:58 XJT 52533070, stringGSL=base_string_998753__00011209, boolGSL=true, sortableGSL=base_string_551223__00007785, _version_=1725140713077735425, _root_=12}, SolrDocument{id=13, intGSL=1002066475, longGSL=267061401255889156, doubleGSL=11119.07322926528, floatGSL=10295.165, dateGSL=Thu Dec 08 09:13:58 XJT 52533070, stringGSL=base_string_998753__00011209, boolGSL=false, sortableGSL=base_string_551223__00007785, _version_=1725140713079832576, _root_=13}, SolrDocument{id=7, intGSL=1002066475, longGSL=267061401255889156, doubleGSL=11119.07322926528, floatGSL=10295.165, dateGSL=Thu Dec 08 09:13:58 XJT 52533070, stringGSL=base_string_998753__00011209, boolGSL=false, sortableGSL=base_string_551223__00007785, _version_=1725140713078784001, _root_=7}, SolrDocument{id=8, intGSL=1002066475, longGSL=267061401255889156, doubleGSL=11119.07322926528, floatGSL=10295.165, dateGSL=Thu Dec 08 09:13:58 XJT 52533070, stringGSL=base_string_998753__00011209, boolGSL=true, sortableGSL=base_string_551223__00007785, _version_=1725140713077735424, _root_=8}, SolrDocument{id=9, intGSL=1002066475, longGSL=267061401255889156, doubleGSL=11119.07322926528, floatGSL=10295.165, dateGSL=Thu Dec 08 09:13:58 XJT 52533070, stringGSL=base_string_998753__00011209, boolGSL=false, sortableGSL=base_string_551223__00007785, _version_=1725140713079832576, _root_=9}]}}, {groupValue=1002072701, doclist={numFound=5,numFoundExact=true,start=0,maxScore=1.0,docs=[SolrDocument{id=14, intGSL=1002072701, longGSL=267061401255894648, doubleGSL=12294.07322926528, floatGSL=14984.165, dateGSL=Thu Dec 08 09:14:00 XJT 52533070, stringGSL=base_string_998753__00017372, boolGSL=true, sortableGSL=base_string_551223__00009830, _version_=1725140713082978304, _root_=14}, SolrDocument{id=15, intGSL=1002072701, longGSL=267061401255894648, doubleGSL=12294.07322926528, floatGSL=14984.165, dateGSL=Thu Dec 08 09:14:00 XJT 52533070, stringGSL=base_string_998753__00017372, boolGSL=false, sortableGSL=base_string_551223__00009830, _version_=1725140713077735426, _root_=15}, SolrDocument{id=16, intGSL=1002072701, longGSL=267061401255894648, doubleGSL=12294.07322926528, floatGSL=14984.165, dateGSL=Thu Dec 08 09:14:00 XJT 52533070, stringGSL=base_string_998753__00017372, boolGSL=true, sortableGSL=base_string_551223__00009830, _version_=1725140713078784000, _root_=16}, SolrDocument{id=18, intGSL=1002072701, longGSL=267061401255894648, doubleGSL=12294.07322926528, floatGSL=14984.165, dateGSL=Thu Dec 08 09:14:00 XJT 52533070, stringGSL=base_string_998753__00017372, boolGSL=true, sortableGSL=base_string_551223__00009830, _version_=1725140713080881153, _root_=18}, SolrDocument{id=20, intGSL=1002072701, longGSL=267061401255894648, doubleGSL=12294.07322926528, floatGSL=14984.165, dateGSL=Thu Dec 08 09:14:00 XJT 52533070, stringGSL=base_string_998753__00017372, boolGSL=true, sortableGSL=base_string_551223__00009830, _version_=1725140713079832576, _root_=20}]}}, {groupValue=1002075300, doclist={numFound=4,numFoundExact=true,start=0,maxScore=1.0,docs=[SolrDocument{id=22, intGSL=1002075300, longGSL=267061401255898482, doubleGSL=16273.07322926528, floatGSL=24685.164, dateGSL=Thu Dec 08 09:14:10 XJT 52533070, stringGSL=base_string_998753__00020157, boolGSL=true, sortableGSL=base_string_551223__00013613, _version_=1725140713079832577, _root_=22}, SolrDocument{id=24, intGSL=1002075300, longGSL=267061401255898482, doubleGSL=16273.07322926528, floatGSL=24685.164, dateGSL=Thu Dec 08 09:14:10 XJT 52533070, stringGSL=base_string_998753__00020157, boolGSL=true, sortableGSL=base_string_551223__00013613, _version_=1725140713082978307, _root_=24}, SolrDocument{id=26, intGSL=1002075300, longGSL=267061401255898482, doubleGSL=16273.07322926528, floatGSL=24685.164, dateGSL=Thu Dec 08 09:14:10 XJT 52533070, stringGSL=base_string_998753__00020157, boolGSL=true, sortableGSL=base_string_551223__00013613, _version_=1725140713080881155, _root_=26}, SolrDocument{id=27, intGSL=1002075300, longGSL=267061401255898482, doubleGSL=16273.07322926528, floatGSL=24685.164, dateGSL=Thu Dec 08 09:14:10 XJT 52533070, stringGSL=base_string_998753__00020157, boolGSL=false, sortableGSL=base_string_551223__00013613, _version_=1725140713082978308, _root_=27}]}}, {groupValue=1002083995, doclist={numFound=5,numFoundExact=true,start=0,maxScore=1.0,docs=[SolrDocument{id=28, intGSL=1002083995, longGSL=267061401255901499, doubleGSL=16812.07322926528, floatGSL=31805.164, dateGSL=Thu Dec 08 09:14:11 XJT 52533070, stringGSL=base_string_998753__00023957, boolGSL=true, sortableGSL=base_string_551223__00021286, _version_=1725140713082978309, _root_=28}, SolrDocument{id=30, intGSL=1002083995, longGSL=267061401255901499, doubleGSL=16812.07322926528, floatGSL=31805.164, dateGSL=Thu Dec 08 09:14:11 XJT 52533070, stringGSL=base_string_998753__00023957, boolGSL=true, sortableGSL=base_string_551223__00021286, _version_=1725140713080881156, _root_=30}, SolrDocument{id=31, intGSL=1002083995, longGSL=267061401255901499, doubleGSL=16812.07322926528, floatGSL=31805.164, dateGSL=Thu Dec 08 09:14:11 XJT 52533070, stringGSL=base_string_998753__00023957, boolGSL=false, sortableGSL=base_string_551223__00021286, _version_=1725140713080881157, _root_=31}, SolrDocument{id=32, intGSL=1002083995, longGSL=267061401255901499, doubleGSL=16812.07322926528, floatGSL=31805.164, dateGSL=Thu Dec 08 09:14:11 XJT 52533070, stringGSL=base_string_998753__00023957, boolGSL=true, sortableGSL=base_string_551223__00021286, _version_=1725140713082978310, _root_=32}, SolrDocument{id=34, intGSL=1002083995, longGSL=267061401255901499, doubleGSL=16812.07322926528, floatGSL=31805.164, dateGSL=Thu Dec 08 09:14:11 XJT 52533070, stringGSL=base_string_998753__00023957, boolGSL=true, sortableGSL=base_string_551223__00021286, _version_=1725140713079832578, _root_=34}]}}, {groupValue=1002087340, doclist={numFound=6,numFoundExact=true,start=0,maxScore=1.0,docs=[SolrDocument{id=35, intGSL=1002087340, longGSL=267061401255905558, doubleGSL=17899.07322926528, floatGSL=32053.164, dateGSL=Thu Dec 08 09:14:18 XJT 52533070, stringGSL=base_string_998753__00033035, boolGSL=false, sortableGSL=base_string_551223__00024779, _version_=1725140713079832579, _root_=35}, SolrDocument{id=36, intGSL=1002087340, longGSL=267061401255905558, doubleGSL=17899.07322926528, floatGSL=32053.164, dateGSL=Thu Dec 08 09:14:18 XJT 52533070, stringGSL=base_string_998753__00033035, boolGSL=true, sortableGSL=base_string_551223__00024779, _version_=1725140713079832580, _root_=36}, SolrDocument{id=38, intGSL=1002087340, longGSL=267061401255905558, doubleGSL=17899.07322926528, floatGSL=32053.164, dateGSL=Thu Dec 08 09:14:18 XJT 52533070, stringGSL=base_string_998753__00033035, boolGSL=true, sortableGSL=base_string_551223__00024779, _version_=1725140713082978311, _root_=38}, SolrDocument{id=39, intGSL=1002087340, longGSL=267061401255905558, doubleGSL=17899.07322926528, floatGSL=32053.164, dateGSL=Thu Dec 08 09:14:18 XJT 52533070, stringGSL=base_string_998753__00033035, boolGSL=false, sortableGSL=base_string_551223__00024779, _version_=1725140713080881159, _root_=39}, SolrDocument{id=40, intGSL=1002087340, longGSL=267061401255905558, doubleGSL=17899.07322926528, floatGSL=32053.164, dateGSL=Thu Dec 08 09:14:18 XJT 52533070, stringGSL=base_string_998753__00033035, boolGSL=true, sortableGSL=base_string_551223__00024779, _version_=1725140713082978312, _root_=40}, SolrDocument{id=41, intGSL=1002087340, longGSL=267061401255905558, doubleGSL=17899.07322926528, floatGSL=32053.164, dateGSL=Thu Dec 08 09:14:18 XJT 52533070, stringGSL=base_string_998753__00033035, boolGSL=false, sortableGSL=base_string_551223__00024779, _version_=1725140713079832581, _root_=41}]}}, {groupValue=1002094820, doclist={numFound=6,numFoundExact=true,start=0,maxScore=1.0,docs=[SolrDocument{id=42, intGSL=1002094820, longGSL=267061401255910036, doubleGSL=20223.07322926528, floatGSL=36366.164, dateGSL=Thu Dec 08 09:14:24 XJT 52533070, stringGSL=base_string_998753__00035431, boolGSL=true, sortableGSL=base_string_551223__00028733, _version_=1725140713082978313, _root_=42}, SolrDocument{id=43, intGSL=1002094820, longGSL=267061401255910036, doubleGSL=20223.07322926528, floatGSL=36366.164, dateGSL=Thu Dec 08 09:14:24 XJT 52533070, stringGSL=base_string_998753__00035431, boolGSL=false, sortableGSL=base_string_551223__00028733, _version_=1725140713079832582, _root_=43}, SolrDocument{id=44, intGSL=1002094820, longGSL=267061401255910036, doubleGSL=20223.07322926528, floatGSL=36366.164, dateGSL=Thu Dec 08 09:14:24 XJT 52533070, stringGSL=base_string_998753__00035431, boolGSL=true, sortableGSL=base_string_551223__00028733, _version_=1725140713079832583, _root_=44}, SolrDocument{id=46, intGSL=1002094820, longGSL=267061401255910036, doubleGSL=20223.07322926528, floatGSL=36366.164, dateGSL=Thu Dec 08 09:14:24 XJT 52533070, stringGSL=base_string_998753__00035431, boolGSL=true, sortableGSL=base_string_551223__00028733, _version_=1725140713079832584, _root_=46}, SolrDocument{id=47, intGSL=1002094820, longGSL=267061401255910036, doubleGSL=20223.07322926528, floatGSL=36366.164, dateGSL=Thu Dec 08 09:14:24 XJT 52533070, stringGSL=base_string_998753__00035431, boolGSL=false, sortableGSL=base_string_551223__00028733, _version_=1725140713079832585, _root_=47}, SolrDocument{id=48, intGSL=1002094820, longGSL=267061401255910036, doubleGSL=20223.07322926528, floatGSL=36366.164, dateGSL=Thu Dec 08 09:14:24 XJT 52533070, stringGSL=base_string_998753__00035431, boolGSL=true, sortableGSL=base_string_551223__00028733, _version_=1725140713082978314, _root_=48}]}}]}}}
	at __randomizedtesting.SeedInfo.seed([2EB95176E42D4766:9B4A1545858A2238]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.solr.cloud.DocValuesNotIndexedTest.doGroupingDvOnly(DocValuesNotIndexedTest.java:403)
	at org.apache.solr.cloud.DocValuesNotIndexedTest.testGroupingDVOnlySortLast(DocValuesNotIndexedTest.java:331)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:829)


FAILED:  org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=4140, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:156)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:89)
        at app//org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:342)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:82)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getLines(AbstractAnalysisFactory.java:302)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getWordSet(AbstractAnalysisFactory.java:293)
        at app//org.apache.lucene.analysis.en.AbstractWordsFileFilterFactory.inform(AbstractWordsFileFilterFactory.java:88)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:723)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1341)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1194)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1179)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:615)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:177)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:107)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:244)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:314)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:241)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$589/0x0000000800695c40.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:46)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:239)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1686)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3134)
        at app//org.apache.solr.core.SolrCore$$Lambda$712/0x0000000800811840.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2499)
        at app//org.apache.solr.cloud.ZkController$$Lambda$820/0x0000000800899040.run(Unknown Source)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=4140, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:156)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:89)
        at app//org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:342)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:82)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getLines(AbstractAnalysisFactory.java:302)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getWordSet(AbstractAnalysisFactory.java:293)
        at app//org.apache.lucene.analysis.en.AbstractWordsFileFilterFactory.inform(AbstractWordsFileFilterFactory.java:88)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:723)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1341)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1194)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1179)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:615)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:177)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:107)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:244)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:314)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:241)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$589/0x0000000800695c40.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:46)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:239)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1686)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3134)
        at app//org.apache.solr.core.SolrCore$$Lambda$712/0x0000000800811840.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2499)
        at app//org.apache.solr.cloud.ZkController$$Lambda$820/0x0000000800899040.run(Unknown Source)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
	at __randomizedtesting.SeedInfo.seed([2EB95176E42D4766]:0)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
java.security.AccessControlException: access denied ("java.io.FilePermission" "/Users/jenkins/workspace/Solr-main-MacOSX/solr/core/build/resources/test/solr/filestore" "write")

Stack Trace:
java.security.AccessControlException: access denied ("java.io.FilePermission" "/Users/jenkins/workspace/Solr-main-MacOSX/solr/core/build/resources/test/solr/filestore" "write")
	at __randomizedtesting.SeedInfo.seed([2EB95176E42D4766]:0)
	at java.base/java.security.AccessControlContext.checkPermission(AccessControlContext.java:472)
	at java.base/java.security.AccessController.checkPermission(AccessController.java:897)
	at java.base/java.lang.SecurityManager.checkPermission(SecurityManager.java:322)
	at java.base/java.lang.SecurityManager.checkWrite(SecurityManager.java:752)
	at java.base/sun.nio.fs.UnixPath.checkWrite(UnixPath.java:824)
	at java.base/sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:381)
	at java.base/java.nio.file.Files.createDirectory(Files.java:690)
	at java.base/java.nio.file.Files.createAndCheckIsDirectory(Files.java:797)
	at java.base/java.nio.file.Files.createDirectories(Files.java:743)
	at org.apache.solr.filestore.DistribPackageStore.getPackageStoreDirPath(DistribPackageStore.java:584)
	at org.apache.solr.filestore.DistribPackageStore.ensurePackageStoreDir(DistribPackageStore.java:567)
	at org.apache.solr.filestore.DistribPackageStore.<init>(DistribPackageStore.java:83)
	at org.apache.solr.filestore.PackageStoreAPI.<init>(PackageStoreAPI.java:76)
	at org.apache.solr.core.CoreContainer.load(CoreContainer.java:725)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:184)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:156)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:162)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:112)
	at org.apache.solr.SolrTestCaseJ4.createCore(SolrTestCaseJ4.java:813)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:803)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:650)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:639)
	at org.apache.solr.cloud.AbstractZkTestCase.azt_beforeClass(AbstractZkTestCase.java:68)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:882)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:829)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 8 threads leaked from SUITE scope at org.apache.solr.rest.TestManagedResourceStorage: 
   1) Thread[id=24593, name=OverseerStateUpdate-72081207751344131-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:294)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   2) Thread[id=24595, name=zkCallback-15947-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   3) Thread[id=24583, name=httpShardExecutor-15941-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$254/0x000000080040c040.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$217/0x00000008003b7040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   4) Thread[id=24590, name=SUITE-TestManagedResourceStorage-seed#[2EB95176E42D4766]-worker-SendThread(127.0.0.1:55345), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:214)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1380)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanAndNotifyState(ClientCnxn.java:1321)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1299)
   5) Thread[id=24591, name=SUITE-TestManagedResourceStorage-seed#[2EB95176E42D4766]-worker-EventThread, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:551)
   6) Thread[id=24592, name=zkConnectionManagerCallback-15948-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   7) Thread[id=24594, name=OverseerCollectionConfigSetProcessor-72081207751344131-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:271)
        at app//org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:258)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   8) Thread[id=24586, name=h2sc-15943-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$254/0x000000080040c040.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$217/0x00000008003b7040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 8 threads leaked from SUITE scope at org.apache.solr.rest.TestManagedResourceStorage: 
   1) Thread[id=24593, name=OverseerStateUpdate-72081207751344131-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:294)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   2) Thread[id=24595, name=zkCallback-15947-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   3) Thread[id=24583, name=httpShardExecutor-15941-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$254/0x000000080040c040.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$217/0x00000008003b7040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   4) Thread[id=24590, name=SUITE-TestManagedResourceStorage-seed#[2EB95176E42D4766]-worker-SendThread(127.0.0.1:55345), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:214)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1380)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanAndNotifyState(ClientCnxn.java:1321)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1299)
   5) Thread[id=24591, name=SUITE-TestManagedResourceStorage-seed#[2EB95176E42D4766]-worker-EventThread, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:551)
   6) Thread[id=24592, name=zkConnectionManagerCallback-15948-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   7) Thread[id=24594, name=OverseerCollectionConfigSetProcessor-72081207751344131-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:271)
        at app//org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:258)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   8) Thread[id=24586, name=h2sc-15943-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$254/0x000000080040c040.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$217/0x00000008003b7040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
	at __randomizedtesting.SeedInfo.seed([2EB95176E42D4766]:0)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=24595, name=zkCallback-15947-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   2) Thread[id=24583, name=httpShardExecutor-15941-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$254/0x000000080040c040.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$217/0x00000008003b7040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   3) Thread[id=24590, name=SUITE-TestManagedResourceStorage-seed#[2EB95176E42D4766]-worker-SendThread(127.0.0.1:55345), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:362)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1197)
   4) Thread[id=24592, name=zkConnectionManagerCallback-15948-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   5) Thread[id=24586, name=h2sc-15943-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$254/0x000000080040c040.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$217/0x00000008003b7040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=24595, name=zkCallback-15947-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   2) Thread[id=24583, name=httpShardExecutor-15941-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$254/0x000000080040c040.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$217/0x00000008003b7040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   3) Thread[id=24590, name=SUITE-TestManagedResourceStorage-seed#[2EB95176E42D4766]-worker-SendThread(127.0.0.1:55345), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:362)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1197)
   4) Thread[id=24592, name=zkConnectionManagerCallback-15948-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   5) Thread[id=24586, name=h2sc-15943-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$254/0x000000080040c040.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$217/0x00000008003b7040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
	at __randomizedtesting.SeedInfo.seed([2EB95176E42D4766]:0)

[JENKINS] Solr-main-MacOSX (64bit/jdk-11.0.13) - Build # 661 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/661/
Java: 64bit/jdk-11.0.13 -XX:-UseCompressedOops -XX:+UseParallelGC

6 tests failed.
FAILED:  org.apache.solr.cloud.DocValuesNotIndexedTest.testGroupingDVOnlySortFirst

Error Message:
java.lang.AssertionError: Unexpected number of elements in the group for 'intGSF' size: '6' GroupValue: '1037799198' rsp: {responseHeader={zkConnected=true, status=0, QTime=17, params={q=*:*, group.sort=id asc, _stateVer_=dv_coll:13, group.limit=100, rows=100, wt=javabin, version=2, group.field=intGSF, group=true}}, grouped={intGSF={matches=47, groups=[{groupValue=1037799198, doclist={numFound=6,numFoundExact=true,start=0,maxScore=1.0,docs=[SolrDocument{id=0, intGSF=1037799198, longGSF=3776617155976753296, doubleGSF=77.15361534279357, floatGSF=4733.19, dateGSF=Fri Oct 14 21:55:17 OMST 133166760, stringGSF=base_string_900831__00007490, boolGSF=true, sortableGSF=base_string_205906__00005253, _version_=1725128632470339584, _root_=0}, SolrDocument{id=1, intGSF=1037799198, longGSF=3776617155976753296, doubleGSF=77.15361534279357, floatGSF=4733.19, dateGSF=Fri Oct 14 21:55:17 OMST 133166760, stringGSF=base_string_900831__00007490, boolGSF=false, sortableGSF=base_string_205906__00005253, _version_=1725128632468242432, _root_=1}, SolrDocument{id=2, intGSF=1037799198, longGSF=3776617155976753296, doubleGSF=77.15361534279357, floatGSF=4733.19, dateGSF=Fri Oct 14 21:55:17 OMST 133166760, stringGSF=base_string_900831__00007490, boolGSF=true, sortableGSF=base_string_205906__00005253, _version_=1725128632471388160, _root_=2}, SolrDocument{id=3, intGSF=1037799198, longGSF=3776617155976753296, doubleGSF=77.15361534279357, floatGSF=4733.19, dateGSF=Fri Oct 14 21:55:17 OMST 133166760, stringGSF=base_string_900831__00007490, boolGSF=false, sortableGSF=base_string_205906__00005253, _version_=1725128632472436736, _root_=3}, SolrDocument{id=4, intGSF=1037799198, longGSF=3776617155976753296, doubleGSF=77.15361534279357, floatGSF=4733.19, dateGSF=Fri Oct 14 21:55:17 OMST 133166760, stringGSF=base_string_900831__00007490, boolGSF=true, sortableGSF=base_string_205906__00005253, _version_=1725128632471388160, _root_=4}, SolrDocument{id=5, intGSF=1037799198, longGSF=3776617155976753296, doubleGSF=77.15361534279357, floatGSF=4733.19, dateGSF=Fri Oct 14 21:55:17 OMST 133166760, stringGSF=base_string_900831__00007490, boolGSF=false, sortableGSF=base_string_205906__00005253, _version_=1725128632469291008, _root_=5}]}}, {groupValue=1037809160, doclist={numFound=5,numFoundExact=true,start=0,maxScore=1.0,docs=[SolrDocument{id=10, intGSF=1037809160, longGSF=3776617155976754011, doubleGSF=273.15361534279356, floatGSF=8907.19, dateGSF=Fri Oct 14 21:55:21 OMST 133166760, stringGSF=base_string_900831__00010291, boolGSF=true, sortableGSF=base_string_205906__00010324, _version_=1725128632469291009, _root_=10}, SolrDocument{id=11, intGSF=1037809160, longGSF=3776617155976754011, doubleGSF=273.15361534279356, floatGSF=8907.19, dateGSF=Fri Oct 14 21:55:21 OMST 133166760, stringGSF=base_string_900831__00010291, boolGSF=false, sortableGSF=base_string_205906__00010324, _version_=1725128632469291010, _root_=11}, SolrDocument{id=12, intGSF=1037809160, longGSF=3776617155976754011, doubleGSF=273.15361534279356, floatGSF=8907.19, dateGSF=Fri Oct 14 21:55:21 OMST 133166760, stringGSF=base_string_900831__00010291, boolGSF=true, sortableGSF=base_string_205906__00010324, _version_=1725128632471388162, _root_=12}, SolrDocument{id=13, intGSF=1037809160, longGSF=3776617155976754011, doubleGSF=273.15361534279356, floatGSF=8907.19, dateGSF=Fri Oct 14 21:55:21 OMST 133166760, stringGSF=base_string_900831__00010291, boolGSF=false, sortableGSF=base_string_205906__00010324, _version_=1725128632469291011, _root_=13}, SolrDocument{id=8, intGSF=1037809160, longGSF=3776617155976754011, doubleGSF=273.15361534279356, floatGSF=8907.19, dateGSF=Fri Oct 14 21:55:21 OMST 133166760, stringGSF=base_string_900831__00010291, boolGSF=true, sortableGSF=base_string_205906__00010324, _version_=1725128632469291008, _root_=8}]}}, {groupValue=1037818559, doclist={numFound=6,numFoundExact=true,start=0,maxScore=1.0,docs=[SolrDocument{id=14, intGSF=1037818559, longGSF=3776617155976759610, doubleGSF=3041.1536153427937, floatGSF=13537.19, dateGSF=Fri Oct 14 21:55:26 OMST 133166760, stringGSF=base_string_900831__00012883, boolGSF=true, sortableGSF=base_string_205906__00013610, _version_=1725128632469291012, _root_=14}, SolrDocument{id=15, intGSF=1037818559, longGSF=3776617155976759610, doubleGSF=3041.1536153427937, floatGSF=13537.19, dateGSF=Fri Oct 14 21:55:26 OMST 133166760, stringGSF=base_string_900831__00012883, boolGSF=false, sortableGSF=base_string_205906__00013610, _version_=1725128632471388163, _root_=15}, SolrDocument{id=16, intGSF=1037818559, longGSF=3776617155976759610, doubleGSF=3041.1536153427937, floatGSF=13537.19, dateGSF=Fri Oct 14 21:55:26 OMST 133166760, stringGSF=base_string_900831__00012883, boolGSF=true, sortableGSF=base_string_205906__00013610, _version_=1725128632471388164, _root_=16}, SolrDocument{id=17, intGSF=1037818559, longGSF=3776617155976759610, doubleGSF=3041.1536153427937, floatGSF=13537.19, dateGSF=Fri Oct 14 21:55:26 OMST 133166760, stringGSF=base_string_900831__00012883, boolGSF=false, sortableGSF=base_string_205906__00013610, _version_=1725128632469291009, _root_=17}, SolrDocument{id=19, intGSF=1037818559, longGSF=3776617155976759610, doubleGSF=3041.1536153427937, floatGSF=13537.19, dateGSF=Fri Oct 14 21:55:26 OMST 133166760, stringGSF=base_string_900831__00012883, boolGSF=false, sortableGSF=base_string_205906__00013610, _version_=1725128632469291010, _root_=19}, SolrDocument{id=20, intGSF=1037818559, longGSF=3776617155976759610, doubleGSF=3041.1536153427937, floatGSF=13537.19, dateGSF=Fri Oct 14 21:55:26 OMST 133166760, stringGSF=base_string_900831__00012883, boolGSF=true, sortableGSF=base_string_205906__00013610, _version_=1725128632471388165, _root_=20}]}}, {groupValue=null, doclist={numFound=6,numFoundExact=true,start=0,maxScore=1.0,docs=[SolrDocument{id=10000, _version_=1725128632468242432, _root_=10000}, SolrDocument{id=10005, _version_=1725128632471388161, _root_=10005}, SolrDocument{id=10015, _version_=1725128632469291013, _root_=10015}, SolrDocument{id=10020, _version_=1725128632469291014, _root_=10020}, SolrDocument{id=10030, _version_=1725128632470339587, _root_=10030}, SolrDocument{id=10040, _version_=1725128632470339590, _root_=10040}]}}, {groupValue=1037828524, doclist={numFound=6,numFoundExact=true,start=0,maxScore=1.0,docs=[SolrDocument{id=21, intGSF=1037828524, longGSF=3776617155976763780, doubleGSF=5523.153615342793, floatGSF=13919.19, dateGSF=Fri Oct 14 21:55:35 OMST 133166760, stringGSF=base_string_900831__00014647, boolGSF=false, sortableGSF=base_string_205906__00022202, _version_=1725128632469291011, _root_=21}, SolrDocument{id=22, intGSF=1037828524, longGSF=3776617155976763780, doubleGSF=5523.153615342793, floatGSF=13919.19, dateGSF=Fri Oct 14 21:55:35 OMST 133166760, stringGSF=base_string_900831__00014647, boolGSF=true, sortableGSF=base_string_205906__00022202, _version_=1725128632471388166, _root_=22}, SolrDocument{id=23, intGSF=1037828524, longGSF=3776617155976763780, doubleGSF=5523.153615342793, floatGSF=13919.19, dateGSF=Fri Oct 14 21:55:35 OMST 133166760, stringGSF=base_string_900831__00014647, boolGSF=false, sortableGSF=base_string_205906__00022202, _version_=1725128632470339584, _root_=23}, SolrDocument{id=24, intGSF=1037828524, longGSF=3776617155976763780, doubleGSF=5523.153615342793, floatGSF=13919.19, dateGSF=Fri Oct 14 21:55:35 OMST 133166760, stringGSF=base_string_900831__00014647, boolGSF=true, sortableGSF=base_string_205906__00022202, _version_=1725128632469291015, _root_=24}, SolrDocument{id=25, intGSF=1037828524, longGSF=3776617155976763780, doubleGSF=5523.153615342793, floatGSF=13919.19, dateGSF=Fri Oct 14 21:55:35 OMST 133166760, stringGSF=base_string_900831__00014647, boolGSF=false, sortableGSF=base_string_205906__00022202, _version_=1725128632470339585, _root_=25}, SolrDocument{id=27, intGSF=1037828524, longGSF=3776617155976763780, doubleGSF=5523.153615342793, floatGSF=13919.19, dateGSF=Fri Oct 14 21:55:35 OMST 133166760, stringGSF=base_string_900831__00014647, boolGSF=false, sortableGSF=base_string_205906__00022202, _version_=1725128632470339584, _root_=27}]}}, {groupValue=1037829513, doclist={numFound=5,numFoundExact=true,start=0,maxScore=1.0,docs=[SolrDocument{id=28, intGSF=1037829513, longGSF=3776617155976767251, doubleGSF=9800.153615342793, floatGSF=17729.19, dateGSF=Fri Oct 14 21:55:36 OMST 133166760, stringGSF=base_string_900831__00022747, boolGSF=true, sortableGSF=base_string_205906__00022803, _version_=1725128632470339585, _root_=28}, SolrDocument{id=29, intGSF=1037829513, longGSF=3776617155976767251, doubleGSF=9800.153615342793, floatGSF=17729.19, dateGSF=Fri Oct 14 21:55:36 OMST 133166760, stringGSF=base_string_900831__00022747, boolGSF=false, sortableGSF=base_string_205906__00022803, _version_=1725128632470339586, _root_=29}, SolrDocument{id=32, intGSF=1037829513, longGSF=3776617155976767251, doubleGSF=9800.153615342793, floatGSF=17729.19, dateGSF=Fri Oct 14 21:55:36 OMST 133166760, stringGSF=base_string_900831__00022747, boolGSF=true, sortableGSF=base_string_205906__00022803, _version_=1725128632470339586, _root_=32}, SolrDocument{id=33, intGSF=1037829513, longGSF=3776617155976767251, doubleGSF=9800.153615342793, floatGSF=17729.19, dateGSF=Fri Oct 14 21:55:36 OMST 133166760, stringGSF=base_string_900831__00022747, boolGSF=false, sortableGSF=base_string_205906__00022803, _version_=1725128632470339588, _root_=33}, SolrDocument{id=34, intGSF=1037829513, longGSF=3776617155976767251, doubleGSF=9800.153615342793, floatGSF=17729.19, dateGSF=Fri Oct 14 21:55:36 OMST 133166760, stringGSF=base_string_900831__00022747, boolGSF=true, sortableGSF=base_string_205906__00022803, _version_=1725128632471388167, _root_=34}]}}, {groupValue=1037837332, doclist={numFound=6,numFoundExact=true,start=0,maxScore=1.0,docs=[SolrDocument{id=35, intGSF=1037837332, longGSF=3776617155976776147, doubleGSF=18513.153615342795, floatGSF=24295.19, dateGSF=Fri Oct 14 21:55:45 OMST 133166760, stringGSF=base_string_900831__00031778, boolGSF=false, sortableGSF=base_string_205906__00024716, _version_=1725128632471388168, _root_=35}, SolrDocument{id=36, intGSF=1037837332, longGSF=3776617155976776147, doubleGSF=18513.153615342795, floatGSF=24295.19, dateGSF=Fri Oct 14 21:55:45 OMST 133166760, stringGSF=base_string_900831__00031778, boolGSF=true, sortableGSF=base_string_205906__00024716, _version_=1725128632471388169, _root_=36}, SolrDocument{id=37, intGSF=1037837332, longGSF=3776617155976776147, doubleGSF=18513.153615342795, floatGSF=24295.19, dateGSF=Fri Oct 14 21:55:45 OMST 133166760, stringGSF=base_string_900831__00031778, boolGSF=false, sortableGSF=base_string_205906__00024716, _version_=1725128632470339589, _root_=37}, SolrDocument{id=38, intGSF=1037837332, longGSF=3776617155976776147, doubleGSF=18513.153615342795, floatGSF=24295.19, dateGSF=Fri Oct 14 21:55:45 OMST 133166760, stringGSF=base_string_900831__00031778, boolGSF=true, sortableGSF=base_string_205906__00024716, _version_=1725128632470339587, _root_=38}, SolrDocument{id=40, intGSF=1037837332, longGSF=3776617155976776147, doubleGSF=18513.153615342795, floatGSF=24295.19, dateGSF=Fri Oct 14 21:55:45 OMST 133166760, stringGSF=base_string_900831__00031778, boolGSF=true, sortableGSF=base_string_205906__00024716, _version_=1725128632470339588, _root_=40}, SolrDocument{id=41, intGSF=1037837332, longGSF=3776617155976776147, doubleGSF=18513.153615342795, floatGSF=24295.19, dateGSF=Fri Oct 14 21:55:45 OMST 133166760, stringGSF=base_string_900831__00031778, boolGSF=false, sortableGSF=base_string_205906__00024716, _version_=1725128632471388170, _root_=41}]}}, {groupValue=1037839263, doclist={numFound=7,numFoundExact=true,start=0,maxScore=1.0,docs=[SolrDocument{id=42, intGSF=1037839263, longGSF=3776617155976784802, doubleGSF=23322.153615342795, floatGSF=30747.19, dateGSF=Fri Oct 14 21:55:47 OMST 133166760, stringGSF=base_string_900831__00033205, boolGSF=true, sortableGSF=base_string_205906__00034686, _version_=1725128632470339589, _root_=42}, SolrDocument{id=43, intGSF=1037839263, longGSF=3776617155976784802, doubleGSF=23322.153615342795, floatGSF=30747.19, dateGSF=Fri Oct 14 21:55:47 OMST 133166760, stringGSF=base_string_900831__00033205, boolGSF=false, sortableGSF=base_string_205906__00034686, _version_=1725128632471388171, _root_=43}, SolrDocument{id=44, intGSF=1037839263, longGSF=3776617155976784802, doubleGSF=23322.153615342795, floatGSF=30747.19, dateGSF=Fri Oct 14 21:55:47 OMST 133166760, stringGSF=base_string_900831__00033205, boolGSF=true, sortableGSF=base_string_205906__00034686, _version_=1725128632471388172, _root_=44}, SolrDocument{id=45, intGSF=1037839263, longGSF=3776617155976784802, doubleGSF=23322.153615342795, floatGSF=30747.19, dateGSF=Fri Oct 14 21:55:47 OMST 133166760, stringGSF=base_string_900831__00033205, boolGSF=false, sortableGSF=base_string_205906__00034686, _version_=1725128632470339591, _root_=45}, SolrDocument{id=46, intGSF=1037839263, longGSF=3776617155976784802, doubleGSF=23322.153615342795, floatGSF=30747.19, dateGSF=Fri Oct 14 21:55:47 OMST 133166760, stringGSF=base_string_900831__00033205, boolGSF=true, sortableGSF=base_string_205906__00034686, _version_=1725128632471388173, _root_=46}, SolrDocument{id=47, intGSF=1037839263, longGSF=3776617155976784802, doubleGSF=23322.153615342795, floatGSF=30747.19, dateGSF=Fri Oct 14 21:55:47 OMST 133166760, stringGSF=base_string_900831__00033205, boolGSF=false, sortableGSF=base_string_205906__00034686, _version_=1725128632471388174, _root_=47}, SolrDocument{id=48, intGSF=1037839263, longGSF=3776617155976784802, doubleGSF=23322.153615342795, floatGSF=30747.19, dateGSF=Fri Oct 14 21:55:47 OMST 133166760, stringGSF=base_string_900831__00033205, boolGSF=true, sortableGSF=base_string_205906__00034686, _version_=1725128632470339590, _root_=48}]}}]}}}

Stack Trace:
java.lang.AssertionError: Unexpected number of elements in the group for 'intGSF' size: '6' GroupValue: '1037799198' rsp: {responseHeader={zkConnected=true, status=0, QTime=17, params={q=*:*, group.sort=id asc, _stateVer_=dv_coll:13, group.limit=100, rows=100, wt=javabin, version=2, group.field=intGSF, group=true}}, grouped={intGSF={matches=47, groups=[{groupValue=1037799198, doclist={numFound=6,numFoundExact=true,start=0,maxScore=1.0,docs=[SolrDocument{id=0, intGSF=1037799198, longGSF=3776617155976753296, doubleGSF=77.15361534279357, floatGSF=4733.19, dateGSF=Fri Oct 14 21:55:17 OMST 133166760, stringGSF=base_string_900831__00007490, boolGSF=true, sortableGSF=base_string_205906__00005253, _version_=1725128632470339584, _root_=0}, SolrDocument{id=1, intGSF=1037799198, longGSF=3776617155976753296, doubleGSF=77.15361534279357, floatGSF=4733.19, dateGSF=Fri Oct 14 21:55:17 OMST 133166760, stringGSF=base_string_900831__00007490, boolGSF=false, sortableGSF=base_string_205906__00005253, _version_=1725128632468242432, _root_=1}, SolrDocument{id=2, intGSF=1037799198, longGSF=3776617155976753296, doubleGSF=77.15361534279357, floatGSF=4733.19, dateGSF=Fri Oct 14 21:55:17 OMST 133166760, stringGSF=base_string_900831__00007490, boolGSF=true, sortableGSF=base_string_205906__00005253, _version_=1725128632471388160, _root_=2}, SolrDocument{id=3, intGSF=1037799198, longGSF=3776617155976753296, doubleGSF=77.15361534279357, floatGSF=4733.19, dateGSF=Fri Oct 14 21:55:17 OMST 133166760, stringGSF=base_string_900831__00007490, boolGSF=false, sortableGSF=base_string_205906__00005253, _version_=1725128632472436736, _root_=3}, SolrDocument{id=4, intGSF=1037799198, longGSF=3776617155976753296, doubleGSF=77.15361534279357, floatGSF=4733.19, dateGSF=Fri Oct 14 21:55:17 OMST 133166760, stringGSF=base_string_900831__00007490, boolGSF=true, sortableGSF=base_string_205906__00005253, _version_=1725128632471388160, _root_=4}, SolrDocument{id=5, intGSF=1037799198, longGSF=3776617155976753296, doubleGSF=77.15361534279357, floatGSF=4733.19, dateGSF=Fri Oct 14 21:55:17 OMST 133166760, stringGSF=base_string_900831__00007490, boolGSF=false, sortableGSF=base_string_205906__00005253, _version_=1725128632469291008, _root_=5}]}}, {groupValue=1037809160, doclist={numFound=5,numFoundExact=true,start=0,maxScore=1.0,docs=[SolrDocument{id=10, intGSF=1037809160, longGSF=3776617155976754011, doubleGSF=273.15361534279356, floatGSF=8907.19, dateGSF=Fri Oct 14 21:55:21 OMST 133166760, stringGSF=base_string_900831__00010291, boolGSF=true, sortableGSF=base_string_205906__00010324, _version_=1725128632469291009, _root_=10}, SolrDocument{id=11, intGSF=1037809160, longGSF=3776617155976754011, doubleGSF=273.15361534279356, floatGSF=8907.19, dateGSF=Fri Oct 14 21:55:21 OMST 133166760, stringGSF=base_string_900831__00010291, boolGSF=false, sortableGSF=base_string_205906__00010324, _version_=1725128632469291010, _root_=11}, SolrDocument{id=12, intGSF=1037809160, longGSF=3776617155976754011, doubleGSF=273.15361534279356, floatGSF=8907.19, dateGSF=Fri Oct 14 21:55:21 OMST 133166760, stringGSF=base_string_900831__00010291, boolGSF=true, sortableGSF=base_string_205906__00010324, _version_=1725128632471388162, _root_=12}, SolrDocument{id=13, intGSF=1037809160, longGSF=3776617155976754011, doubleGSF=273.15361534279356, floatGSF=8907.19, dateGSF=Fri Oct 14 21:55:21 OMST 133166760, stringGSF=base_string_900831__00010291, boolGSF=false, sortableGSF=base_string_205906__00010324, _version_=1725128632469291011, _root_=13}, SolrDocument{id=8, intGSF=1037809160, longGSF=3776617155976754011, doubleGSF=273.15361534279356, floatGSF=8907.19, dateGSF=Fri Oct 14 21:55:21 OMST 133166760, stringGSF=base_string_900831__00010291, boolGSF=true, sortableGSF=base_string_205906__00010324, _version_=1725128632469291008, _root_=8}]}}, {groupValue=1037818559, doclist={numFound=6,numFoundExact=true,start=0,maxScore=1.0,docs=[SolrDocument{id=14, intGSF=1037818559, longGSF=3776617155976759610, doubleGSF=3041.1536153427937, floatGSF=13537.19, dateGSF=Fri Oct 14 21:55:26 OMST 133166760, stringGSF=base_string_900831__00012883, boolGSF=true, sortableGSF=base_string_205906__00013610, _version_=1725128632469291012, _root_=14}, SolrDocument{id=15, intGSF=1037818559, longGSF=3776617155976759610, doubleGSF=3041.1536153427937, floatGSF=13537.19, dateGSF=Fri Oct 14 21:55:26 OMST 133166760, stringGSF=base_string_900831__00012883, boolGSF=false, sortableGSF=base_string_205906__00013610, _version_=1725128632471388163, _root_=15}, SolrDocument{id=16, intGSF=1037818559, longGSF=3776617155976759610, doubleGSF=3041.1536153427937, floatGSF=13537.19, dateGSF=Fri Oct 14 21:55:26 OMST 133166760, stringGSF=base_string_900831__00012883, boolGSF=true, sortableGSF=base_string_205906__00013610, _version_=1725128632471388164, _root_=16}, SolrDocument{id=17, intGSF=1037818559, longGSF=3776617155976759610, doubleGSF=3041.1536153427937, floatGSF=13537.19, dateGSF=Fri Oct 14 21:55:26 OMST 133166760, stringGSF=base_string_900831__00012883, boolGSF=false, sortableGSF=base_string_205906__00013610, _version_=1725128632469291009, _root_=17}, SolrDocument{id=19, intGSF=1037818559, longGSF=3776617155976759610, doubleGSF=3041.1536153427937, floatGSF=13537.19, dateGSF=Fri Oct 14 21:55:26 OMST 133166760, stringGSF=base_string_900831__00012883, boolGSF=false, sortableGSF=base_string_205906__00013610, _version_=1725128632469291010, _root_=19}, SolrDocument{id=20, intGSF=1037818559, longGSF=3776617155976759610, doubleGSF=3041.1536153427937, floatGSF=13537.19, dateGSF=Fri Oct 14 21:55:26 OMST 133166760, stringGSF=base_string_900831__00012883, boolGSF=true, sortableGSF=base_string_205906__00013610, _version_=1725128632471388165, _root_=20}]}}, {groupValue=null, doclist={numFound=6,numFoundExact=true,start=0,maxScore=1.0,docs=[SolrDocument{id=10000, _version_=1725128632468242432, _root_=10000}, SolrDocument{id=10005, _version_=1725128632471388161, _root_=10005}, SolrDocument{id=10015, _version_=1725128632469291013, _root_=10015}, SolrDocument{id=10020, _version_=1725128632469291014, _root_=10020}, SolrDocument{id=10030, _version_=1725128632470339587, _root_=10030}, SolrDocument{id=10040, _version_=1725128632470339590, _root_=10040}]}}, {groupValue=1037828524, doclist={numFound=6,numFoundExact=true,start=0,maxScore=1.0,docs=[SolrDocument{id=21, intGSF=1037828524, longGSF=3776617155976763780, doubleGSF=5523.153615342793, floatGSF=13919.19, dateGSF=Fri Oct 14 21:55:35 OMST 133166760, stringGSF=base_string_900831__00014647, boolGSF=false, sortableGSF=base_string_205906__00022202, _version_=1725128632469291011, _root_=21}, SolrDocument{id=22, intGSF=1037828524, longGSF=3776617155976763780, doubleGSF=5523.153615342793, floatGSF=13919.19, dateGSF=Fri Oct 14 21:55:35 OMST 133166760, stringGSF=base_string_900831__00014647, boolGSF=true, sortableGSF=base_string_205906__00022202, _version_=1725128632471388166, _root_=22}, SolrDocument{id=23, intGSF=1037828524, longGSF=3776617155976763780, doubleGSF=5523.153615342793, floatGSF=13919.19, dateGSF=Fri Oct 14 21:55:35 OMST 133166760, stringGSF=base_string_900831__00014647, boolGSF=false, sortableGSF=base_string_205906__00022202, _version_=1725128632470339584, _root_=23}, SolrDocument{id=24, intGSF=1037828524, longGSF=3776617155976763780, doubleGSF=5523.153615342793, floatGSF=13919.19, dateGSF=Fri Oct 14 21:55:35 OMST 133166760, stringGSF=base_string_900831__00014647, boolGSF=true, sortableGSF=base_string_205906__00022202, _version_=1725128632469291015, _root_=24}, SolrDocument{id=25, intGSF=1037828524, longGSF=3776617155976763780, doubleGSF=5523.153615342793, floatGSF=13919.19, dateGSF=Fri Oct 14 21:55:35 OMST 133166760, stringGSF=base_string_900831__00014647, boolGSF=false, sortableGSF=base_string_205906__00022202, _version_=1725128632470339585, _root_=25}, SolrDocument{id=27, intGSF=1037828524, longGSF=3776617155976763780, doubleGSF=5523.153615342793, floatGSF=13919.19, dateGSF=Fri Oct 14 21:55:35 OMST 133166760, stringGSF=base_string_900831__00014647, boolGSF=false, sortableGSF=base_string_205906__00022202, _version_=1725128632470339584, _root_=27}]}}, {groupValue=1037829513, doclist={numFound=5,numFoundExact=true,start=0,maxScore=1.0,docs=[SolrDocument{id=28, intGSF=1037829513, longGSF=3776617155976767251, doubleGSF=9800.153615342793, floatGSF=17729.19, dateGSF=Fri Oct 14 21:55:36 OMST 133166760, stringGSF=base_string_900831__00022747, boolGSF=true, sortableGSF=base_string_205906__00022803, _version_=1725128632470339585, _root_=28}, SolrDocument{id=29, intGSF=1037829513, longGSF=3776617155976767251, doubleGSF=9800.153615342793, floatGSF=17729.19, dateGSF=Fri Oct 14 21:55:36 OMST 133166760, stringGSF=base_string_900831__00022747, boolGSF=false, sortableGSF=base_string_205906__00022803, _version_=1725128632470339586, _root_=29}, SolrDocument{id=32, intGSF=1037829513, longGSF=3776617155976767251, doubleGSF=9800.153615342793, floatGSF=17729.19, dateGSF=Fri Oct 14 21:55:36 OMST 133166760, stringGSF=base_string_900831__00022747, boolGSF=true, sortableGSF=base_string_205906__00022803, _version_=1725128632470339586, _root_=32}, SolrDocument{id=33, intGSF=1037829513, longGSF=3776617155976767251, doubleGSF=9800.153615342793, floatGSF=17729.19, dateGSF=Fri Oct 14 21:55:36 OMST 133166760, stringGSF=base_string_900831__00022747, boolGSF=false, sortableGSF=base_string_205906__00022803, _version_=1725128632470339588, _root_=33}, SolrDocument{id=34, intGSF=1037829513, longGSF=3776617155976767251, doubleGSF=9800.153615342793, floatGSF=17729.19, dateGSF=Fri Oct 14 21:55:36 OMST 133166760, stringGSF=base_string_900831__00022747, boolGSF=true, sortableGSF=base_string_205906__00022803, _version_=1725128632471388167, _root_=34}]}}, {groupValue=1037837332, doclist={numFound=6,numFoundExact=true,start=0,maxScore=1.0,docs=[SolrDocument{id=35, intGSF=1037837332, longGSF=3776617155976776147, doubleGSF=18513.153615342795, floatGSF=24295.19, dateGSF=Fri Oct 14 21:55:45 OMST 133166760, stringGSF=base_string_900831__00031778, boolGSF=false, sortableGSF=base_string_205906__00024716, _version_=1725128632471388168, _root_=35}, SolrDocument{id=36, intGSF=1037837332, longGSF=3776617155976776147, doubleGSF=18513.153615342795, floatGSF=24295.19, dateGSF=Fri Oct 14 21:55:45 OMST 133166760, stringGSF=base_string_900831__00031778, boolGSF=true, sortableGSF=base_string_205906__00024716, _version_=1725128632471388169, _root_=36}, SolrDocument{id=37, intGSF=1037837332, longGSF=3776617155976776147, doubleGSF=18513.153615342795, floatGSF=24295.19, dateGSF=Fri Oct 14 21:55:45 OMST 133166760, stringGSF=base_string_900831__00031778, boolGSF=false, sortableGSF=base_string_205906__00024716, _version_=1725128632470339589, _root_=37}, SolrDocument{id=38, intGSF=1037837332, longGSF=3776617155976776147, doubleGSF=18513.153615342795, floatGSF=24295.19, dateGSF=Fri Oct 14 21:55:45 OMST 133166760, stringGSF=base_string_900831__00031778, boolGSF=true, sortableGSF=base_string_205906__00024716, _version_=1725128632470339587, _root_=38}, SolrDocument{id=40, intGSF=1037837332, longGSF=3776617155976776147, doubleGSF=18513.153615342795, floatGSF=24295.19, dateGSF=Fri Oct 14 21:55:45 OMST 133166760, stringGSF=base_string_900831__00031778, boolGSF=true, sortableGSF=base_string_205906__00024716, _version_=1725128632470339588, _root_=40}, SolrDocument{id=41, intGSF=1037837332, longGSF=3776617155976776147, doubleGSF=18513.153615342795, floatGSF=24295.19, dateGSF=Fri Oct 14 21:55:45 OMST 133166760, stringGSF=base_string_900831__00031778, boolGSF=false, sortableGSF=base_string_205906__00024716, _version_=1725128632471388170, _root_=41}]}}, {groupValue=1037839263, doclist={numFound=7,numFoundExact=true,start=0,maxScore=1.0,docs=[SolrDocument{id=42, intGSF=1037839263, longGSF=3776617155976784802, doubleGSF=23322.153615342795, floatGSF=30747.19, dateGSF=Fri Oct 14 21:55:47 OMST 133166760, stringGSF=base_string_900831__00033205, boolGSF=true, sortableGSF=base_string_205906__00034686, _version_=1725128632470339589, _root_=42}, SolrDocument{id=43, intGSF=1037839263, longGSF=3776617155976784802, doubleGSF=23322.153615342795, floatGSF=30747.19, dateGSF=Fri Oct 14 21:55:47 OMST 133166760, stringGSF=base_string_900831__00033205, boolGSF=false, sortableGSF=base_string_205906__00034686, _version_=1725128632471388171, _root_=43}, SolrDocument{id=44, intGSF=1037839263, longGSF=3776617155976784802, doubleGSF=23322.153615342795, floatGSF=30747.19, dateGSF=Fri Oct 14 21:55:47 OMST 133166760, stringGSF=base_string_900831__00033205, boolGSF=true, sortableGSF=base_string_205906__00034686, _version_=1725128632471388172, _root_=44}, SolrDocument{id=45, intGSF=1037839263, longGSF=3776617155976784802, doubleGSF=23322.153615342795, floatGSF=30747.19, dateGSF=Fri Oct 14 21:55:47 OMST 133166760, stringGSF=base_string_900831__00033205, boolGSF=false, sortableGSF=base_string_205906__00034686, _version_=1725128632470339591, _root_=45}, SolrDocument{id=46, intGSF=1037839263, longGSF=3776617155976784802, doubleGSF=23322.153615342795, floatGSF=30747.19, dateGSF=Fri Oct 14 21:55:47 OMST 133166760, stringGSF=base_string_900831__00033205, boolGSF=true, sortableGSF=base_string_205906__00034686, _version_=1725128632471388173, _root_=46}, SolrDocument{id=47, intGSF=1037839263, longGSF=3776617155976784802, doubleGSF=23322.153615342795, floatGSF=30747.19, dateGSF=Fri Oct 14 21:55:47 OMST 133166760, stringGSF=base_string_900831__00033205, boolGSF=false, sortableGSF=base_string_205906__00034686, _version_=1725128632471388174, _root_=47}, SolrDocument{id=48, intGSF=1037839263, longGSF=3776617155976784802, doubleGSF=23322.153615342795, floatGSF=30747.19, dateGSF=Fri Oct 14 21:55:47 OMST 133166760, stringGSF=base_string_900831__00033205, boolGSF=true, sortableGSF=base_string_205906__00034686, _version_=1725128632470339590, _root_=48}]}}]}}}
	at __randomizedtesting.SeedInfo.seed([44BB1B812E2AFB0:EC216D4423F6839]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.solr.cloud.DocValuesNotIndexedTest.doGroupingDvOnly(DocValuesNotIndexedTest.java:403)
	at org.apache.solr.cloud.DocValuesNotIndexedTest.testGroupingDVOnlySortFirst(DocValuesNotIndexedTest.java:326)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:829)


FAILED:  org.apache.solr.core.TestLazyCores.testDontEvictUsedCore

Error Message:
java.lang.AssertionError

Stack Trace:
java.lang.AssertionError
	at __randomizedtesting.SeedInfo.seed([44BB1B812E2AFB0:73D870614957F734]:0)
	at org.junit.Assert.fail(Assert.java:87)
	at org.junit.Assert.assertTrue(Assert.java:42)
	at org.junit.Assert.assertTrue(Assert.java:53)
	at org.apache.solr.core.TestLazyCores.testDontEvictUsedCore(TestLazyCores.java:937)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:829)


FAILED:  org.apache.solr.schema.TestBulkSchemaConcurrent.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 8 threads leaked from SUITE scope at org.apache.solr.schema.TestBulkSchemaConcurrent: 
   1) Thread[id=14545, name=ZKEventListenerThread, state=RUNNABLE, group=TGRP-TestBulkSchemaConcurrent]
        at java.base@11.0.13/java.lang.StackTraceElement.initStackTraceElements(Native Method)
        at java.base@11.0.13/java.lang.StackTraceElement.of(StackTraceElement.java:526)
        at java.base@11.0.13/java.lang.Throwable.getOurStackTrace(Throwable.java:828)
        at java.base@11.0.13/java.lang.Throwable.printStackTrace(Throwable.java:660)
        at java.base@11.0.13/java.lang.Throwable.printStackTrace(Throwable.java:725)
        at app//org.apache.logging.log4j.core.pattern.ThrowablePatternConverter.formatOption(ThrowablePatternConverter.java:195)
        at app//org.apache.logging.log4j.core.pattern.ThrowablePatternConverter.format(ThrowablePatternConverter.java:137)
        at app//org.apache.logging.log4j.core.pattern.PatternFormatter.format(PatternFormatter.java:44)
        at app//org.apache.logging.log4j.core.pattern.VariablesNotEmptyReplacementConverter.format(VariablesNotEmptyReplacementConverter.java:83)
        at app//org.apache.logging.log4j.core.pattern.PatternFormatter.format(PatternFormatter.java:44)
        at app//org.apache.logging.log4j.core.pattern.MaxLengthConverter.format(MaxLengthConverter.java:92)
        at app//org.apache.logging.log4j.core.layout.PatternLayout$NoFormatPatternSerializer.toSerializable(PatternLayout.java:343)
        at app//org.apache.logging.log4j.core.layout.PatternLayout.toText(PatternLayout.java:241)
        at app//org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:226)
        at app//org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:60)
        at app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:197)
        at app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:190)
        at app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:181)
        at app//org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:161)
        at app//org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:134)
        at app//org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:125)
        at app//org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:89)
        at app//org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:542)
        at app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:500)
        at app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:483)
        at app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:417)
        at app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82)
        at app//org.apache.logging.log4j.core.Logger.log(Logger.java:161)
        at app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205)
        at app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159)
        at app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142)
        at app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2017)
        at app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1983)
        at app//org.apache.logging.slf4j.Log4jLogger.error(Log4jLogger.java:319)
        at app//org.apache.solr.update.SolrIndexWriter.close(SolrIndexWriter.java:303)
        at app//org.apache.solr.update.DirectUpdateHandler2.closeWriter(DirectUpdateHandler2.java:895)
        at app//org.apache.solr.update.DefaultSolrCoreState.closeIndexWriter(DefaultSolrCoreState.java:96)
        at app//org.apache.solr.update.DefaultSolrCoreState.close(DefaultSolrCoreState.java:393)
        at app//org.apache.solr.update.SolrCoreState.decrefSolrCoreState(SolrCoreState.java:112)
        at app//org.apache.solr.core.SolrCore.doClose(SolrCore.java:1639)
        at app//org.apache.solr.core.SolrCore.close(SolrCore.java:1582)
        at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1100)
        at app//org.apache.solr.core.SolrCore.reload(SolrCore.java:744)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1690)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3134)
        at app//org.apache.solr.core.SolrCore$$Lambda$729/0x000000014d2695b8.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2499)
        at app//org.apache.solr.cloud.ZkController$$Lambda$849/0x000000014daca0b0.run(Unknown Source)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   2) Thread[id=14550, name=ZKEventListenerThread, state=RUNNABLE, group=TGRP-TestBulkSchemaConcurrent]
        at java.base@11.0.13/java.lang.Class.getProtectionDomain0(Native Method)
        at java.base@11.0.13/java.lang.Class.getProtectionDomain(Class.java:2799)
        at app//org.apache.logging.log4j.core.impl.ThrowableProxyHelper.toCacheEntry(ThrowableProxyHelper.java:161)
        at app//org.apache.logging.log4j.core.impl.ThrowableProxyHelper.toExtendedStackTrace(ThrowableProxyHelper.java:98)
        at app//org.apache.logging.log4j.core.impl.ThrowableProxy.<init>(ThrowableProxy.java:112)
        at app//org.apache.logging.log4j.core.impl.ThrowableProxy.<init>(ThrowableProxy.java:96)
        at app//org.apache.logging.log4j.core.impl.Log4jLogEvent.getThrownProxy(Log4jLogEvent.java:629)
        at app//org.apache.logging.log4j.core.pattern.ExtendedThrowablePatternConverter.format(ExtendedThrowablePatternConverter.java:63)
        at app//org.apache.logging.log4j.core.layout.PatternLayout$NoFormatPatternSerializer.toSerializable(PatternLayout.java:343)
        at app//org.apache.logging.log4j.core.layout.PatternLayout.toText(PatternLayout.java:241)
        at app//org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:226)
        at app//org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:60)
        at app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:197)
        at app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:190)
        at app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:181)
        at app//org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:161)
        at app//org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:134)
        at app//org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:125)
        at app//org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:89)
        at app//org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:542)
        at app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:500)
        at app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:483)
        at app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:417)
        at app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82)
        at app//org.apache.logging.log4j.core.Logger.log(Logger.java:161)
        at app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205)
        at app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159)
        at app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142)
        at app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2017)
        at app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1983)
        at app//org.apache.logging.slf4j.Log4jLogger.error(Log4jLogger.java:319)
        at app//org.apache.solr.update.SolrIndexWriter.close(SolrIndexWriter.java:303)
        at app//org.apache.solr.update.DirectUpdateHandler2.closeWriter(DirectUpdateHandler2.java:895)
        at app//org.apache.solr.update.DefaultSolrCoreState.closeIndexWriter(DefaultSolrCoreState.java:96)
        at app//org.apache.solr.update.DefaultSolrCoreState.close(DefaultSolrCoreState.java:393)
        at app//org.apache.solr.update.SolrCoreState.decrefSolrCoreState(SolrCoreState.java:112)
        at app//org.apache.solr.core.SolrCore.doClose(SolrCore.java:1639)
        at app//org.apache.solr.core.SolrCore.close(SolrCore.java:1582)
        at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1100)
        at app//org.apache.solr.core.SolrCore.reload(SolrCore.java:744)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1690)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3134)
        at app//org.apache.solr.core.SolrCore$$Lambda$729/0x000000014d2695b8.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2499)
        at app//org.apache.solr.cloud.ZkController$$Lambda$849/0x000000014daca0b0.run(Unknown Source)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   3) Thread[id=14611, name=searcherExecutor-9501-thread-1, state=WAITING, group=TGRP-TestBulkSchemaConcurrent]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   4) Thread[id=14546, name=ZKEventListenerThread, state=RUNNABLE, group=TGRP-TestBulkSchemaConcurrent]
        at app//org.apache.logging.log4j.core.impl.ThrowableProxyHelper.toExtendedStackTrace(ThrowableProxyHelper.java:98)
        at app//org.apache.logging.log4j.core.impl.ThrowableProxy.<init>(ThrowableProxy.java:112)
        at app//org.apache.logging.log4j.core.impl.ThrowableProxy.<init>(ThrowableProxy.java:96)
        at app//org.apache.logging.log4j.core.impl.Log4jLogEvent.getThrownProxy(Log4jLogEvent.java:629)
        at app//org.apache.logging.log4j.core.pattern.ExtendedThrowablePatternConverter.format(ExtendedThrowablePatternConverter.java:63)
        at app//org.apache.logging.log4j.core.layout.PatternLayout$NoFormatPatternSerializer.toSerializable(PatternLayout.java:343)
        at app//org.apache.logging.log4j.core.layout.PatternLayout.toText(PatternLayout.java:241)
        at app//org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:226)
        at app//org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:60)
        at app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:197)
        at app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:190)
        at app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:181)
        at app//org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:161)
        at app//org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:134)
        at app//org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:125)
        at app//org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:89)
        at app//org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:542)
        at app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:500)
        at app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:483)
        at app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:417)
        at app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82)
        at app//org.apache.logging.log4j.core.Logger.log(Logger.java:161)
        at app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205)
        at app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159)
        at app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142)
        at app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2017)
        at app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1983)
        at app//org.apache.logging.slf4j.Log4jLogger.error(Log4jLogger.java:319)
        at app//org.apache.solr.update.SolrIndexWriter.close(SolrIndexWriter.java:303)
        at app//org.apache.solr.update.DirectUpdateHandler2.closeWriter(DirectUpdateHandler2.java:895)
        at app//org.apache.solr.update.DefaultSolrCoreState.closeIndexWriter(DefaultSolrCoreState.java:96)
        at app//org.apache.solr.update.DefaultSolrCoreState.close(DefaultSolrCoreState.java:393)
        at app//org.apache.solr.update.SolrCoreState.decrefSolrCoreState(SolrCoreState.java:112)
        at app//org.apache.solr.core.SolrCore.doClose(SolrCore.java:1639)
        at app//org.apache.solr.core.SolrCore.close(SolrCore.java:1582)
        at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1100)
        at app//org.apache.solr.core.SolrCore.reload(SolrCore.java:744)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1690)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3134)
        at app//org.apache.solr.core.SolrCore$$Lambda$729/0x000000014d2695b8.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2499)
        at app//org.apache.solr.cloud.ZkController$$Lambda$849/0x000000014daca0b0.run(Unknown Source)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   5) Thread[id=14544, name=ZKEventListenerThread, state=RUNNABLE, group=TGRP-TestBulkSchemaConcurrent]
        at java.base@11.0.13/java.lang.StackTraceElement.initStackTraceElements(Native Method)
        at java.base@11.0.13/java.lang.StackTraceElement.of(StackTraceElement.java:526)
        at java.base@11.0.13/java.lang.Throwable.getOurStackTrace(Throwable.java:828)
        at java.base@11.0.13/java.lang.Throwable.printStackTrace(Throwable.java:660)
        at java.base@11.0.13/java.lang.Throwable.printStackTrace(Throwable.java:725)
        at app//org.apache.logging.log4j.core.pattern.ThrowablePatternConverter.formatOption(ThrowablePatternConverter.java:195)
        at app//org.apache.logging.log4j.core.pattern.ThrowablePatternConverter.format(ThrowablePatternConverter.java:137)
        at app//org.apache.logging.log4j.core.pattern.PatternFormatter.format(PatternFormatter.java:44)
        at app//org.apache.logging.log4j.core.pattern.VariablesNotEmptyReplacementConverter.format(VariablesNotEmptyReplacementConverter.java:83)
        at app//org.apache.logging.log4j.core.pattern.PatternFormatter.format(PatternFormatter.java:44)
        at app//org.apache.logging.log4j.core.pattern.MaxLengthConverter.format(MaxLengthConverter.java:92)
        at app//org.apache.logging.log4j.core.layout.PatternLayout$NoFormatPatternSerializer.toSerializable(PatternLayout.java:343)
        at app//org.apache.logging.log4j.core.layout.PatternLayout.toText(PatternLayout.java:241)
        at app//org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:226)
        at app//org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:60)
        at app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:197)
        at app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:190)
        at app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:181)
        at app//org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:161)
        at app//org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:134)
        at app//org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:125)
        at app//org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:89)
        at app//org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:542)
        at app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:500)
        at app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:483)
        at app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:417)
        at app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82)
        at app//org.apache.logging.log4j.core.Logger.log(Logger.java:161)
        at app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205)
        at app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159)
        at app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142)
        at app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2017)
        at app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1983)
        at app//org.apache.logging.slf4j.Log4jLogger.error(Log4jLogger.java:319)
        at app//org.apache.solr.update.SolrIndexWriter.close(SolrIndexWriter.java:303)
        at app//org.apache.solr.update.DirectUpdateHandler2.closeWriter(DirectUpdateHandler2.java:895)
        at app//org.apache.solr.update.DefaultSolrCoreState.closeIndexWriter(DefaultSolrCoreState.java:96)
        at app//org.apache.solr.update.DefaultSolrCoreState.close(DefaultSolrCoreState.java:393)
        at app//org.apache.solr.update.SolrCoreState.decrefSolrCoreState(SolrCoreState.java:112)
        at app//org.apache.solr.core.SolrCore.doClose(SolrCore.java:1639)
        at app//org.apache.solr.core.SolrCore.close(SolrCore.java:1582)
        at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1100)
        at app//org.apache.solr.core.SolrCore.reload(SolrCore.java:744)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1690)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3134)
        at app//org.apache.solr.core.SolrCore$$Lambda$729/0x000000014d2695b8.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2499)
        at app//org.apache.solr.cloud.ZkController$$Lambda$849/0x000000014daca0b0.run(Unknown Source)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   6) Thread[id=14613, name=searcherExecutor-9509-thread-1, state=WAITING, group=TGRP-TestBulkSchemaConcurrent]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   7) Thread[id=14614, name=searcherExecutor-9503-thread-1, state=WAITING, group=TGRP-TestBulkSchemaConcurrent]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   8) Thread[id=14603, name=searcherExecutor-9505-thread-1, state=WAITING, group=TGRP-TestBulkSchemaConcurrent]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 8 threads leaked from SUITE scope at org.apache.solr.schema.TestBulkSchemaConcurrent: 
   1) Thread[id=14545, name=ZKEventListenerThread, state=RUNNABLE, group=TGRP-TestBulkSchemaConcurrent]
        at java.base@11.0.13/java.lang.StackTraceElement.initStackTraceElements(Native Method)
        at java.base@11.0.13/java.lang.StackTraceElement.of(StackTraceElement.java:526)
        at java.base@11.0.13/java.lang.Throwable.getOurStackTrace(Throwable.java:828)
        at java.base@11.0.13/java.lang.Throwable.printStackTrace(Throwable.java:660)
        at java.base@11.0.13/java.lang.Throwable.printStackTrace(Throwable.java:725)
        at app//org.apache.logging.log4j.core.pattern.ThrowablePatternConverter.formatOption(ThrowablePatternConverter.java:195)
        at app//org.apache.logging.log4j.core.pattern.ThrowablePatternConverter.format(ThrowablePatternConverter.java:137)
        at app//org.apache.logging.log4j.core.pattern.PatternFormatter.format(PatternFormatter.java:44)
        at app//org.apache.logging.log4j.core.pattern.VariablesNotEmptyReplacementConverter.format(VariablesNotEmptyReplacementConverter.java:83)
        at app//org.apache.logging.log4j.core.pattern.PatternFormatter.format(PatternFormatter.java:44)
        at app//org.apache.logging.log4j.core.pattern.MaxLengthConverter.format(MaxLengthConverter.java:92)
        at app//org.apache.logging.log4j.core.layout.PatternLayout$NoFormatPatternSerializer.toSerializable(PatternLayout.java:343)
        at app//org.apache.logging.log4j.core.layout.PatternLayout.toText(PatternLayout.java:241)
        at app//org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:226)
        at app//org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:60)
        at app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:197)
        at app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:190)
        at app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:181)
        at app//org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:161)
        at app//org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:134)
        at app//org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:125)
        at app//org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:89)
        at app//org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:542)
        at app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:500)
        at app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:483)
        at app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:417)
        at app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82)
        at app//org.apache.logging.log4j.core.Logger.log(Logger.java:161)
        at app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205)
        at app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159)
        at app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142)
        at app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2017)
        at app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1983)
        at app//org.apache.logging.slf4j.Log4jLogger.error(Log4jLogger.java:319)
        at app//org.apache.solr.update.SolrIndexWriter.close(SolrIndexWriter.java:303)
        at app//org.apache.solr.update.DirectUpdateHandler2.closeWriter(DirectUpdateHandler2.java:895)
        at app//org.apache.solr.update.DefaultSolrCoreState.closeIndexWriter(DefaultSolrCoreState.java:96)
        at app//org.apache.solr.update.DefaultSolrCoreState.close(DefaultSolrCoreState.java:393)
        at app//org.apache.solr.update.SolrCoreState.decrefSolrCoreState(SolrCoreState.java:112)
        at app//org.apache.solr.core.SolrCore.doClose(SolrCore.java:1639)
        at app//org.apache.solr.core.SolrCore.close(SolrCore.java:1582)
        at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1100)
        at app//org.apache.solr.core.SolrCore.reload(SolrCore.java:744)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1690)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3134)
        at app//org.apache.solr.core.SolrCore$$Lambda$729/0x000000014d2695b8.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2499)
        at app//org.apache.solr.cloud.ZkController$$Lambda$849/0x000000014daca0b0.run(Unknown Source)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   2) Thread[id=14550, name=ZKEventListenerThread, state=RUNNABLE, group=TGRP-TestBulkSchemaConcurrent]
        at java.base@11.0.13/java.lang.Class.getProtectionDomain0(Native Method)
        at java.base@11.0.13/java.lang.Class.getProtectionDomain(Class.java:2799)
        at app//org.apache.logging.log4j.core.impl.ThrowableProxyHelper.toCacheEntry(ThrowableProxyHelper.java:161)
        at app//org.apache.logging.log4j.core.impl.ThrowableProxyHelper.toExtendedStackTrace(ThrowableProxyHelper.java:98)
        at app//org.apache.logging.log4j.core.impl.ThrowableProxy.<init>(ThrowableProxy.java:112)
        at app//org.apache.logging.log4j.core.impl.ThrowableProxy.<init>(ThrowableProxy.java:96)
        at app//org.apache.logging.log4j.core.impl.Log4jLogEvent.getThrownProxy(Log4jLogEvent.java:629)
        at app//org.apache.logging.log4j.core.pattern.ExtendedThrowablePatternConverter.format(ExtendedThrowablePatternConverter.java:63)
        at app//org.apache.logging.log4j.core.layout.PatternLayout$NoFormatPatternSerializer.toSerializable(PatternLayout.java:343)
        at app//org.apache.logging.log4j.core.layout.PatternLayout.toText(PatternLayout.java:241)
        at app//org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:226)
        at app//org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:60)
        at app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:197)
        at app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:190)
        at app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:181)
        at app//org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:161)
        at app//org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:134)
        at app//org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:125)
        at app//org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:89)
        at app//org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:542)
        at app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:500)
        at app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:483)
        at app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:417)
        at app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82)
        at app//org.apache.logging.log4j.core.Logger.log(Logger.java:161)
        at app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205)
        at app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159)
        at app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142)
        at app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2017)
        at app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1983)
        at app//org.apache.logging.slf4j.Log4jLogger.error(Log4jLogger.java:319)
        at app//org.apache.solr.update.SolrIndexWriter.close(SolrIndexWriter.java:303)
        at app//org.apache.solr.update.DirectUpdateHandler2.closeWriter(DirectUpdateHandler2.java:895)
        at app//org.apache.solr.update.DefaultSolrCoreState.closeIndexWriter(DefaultSolrCoreState.java:96)
        at app//org.apache.solr.update.DefaultSolrCoreState.close(DefaultSolrCoreState.java:393)
        at app//org.apache.solr.update.SolrCoreState.decrefSolrCoreState(SolrCoreState.java:112)
        at app//org.apache.solr.core.SolrCore.doClose(SolrCore.java:1639)
        at app//org.apache.solr.core.SolrCore.close(SolrCore.java:1582)
        at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1100)
        at app//org.apache.solr.core.SolrCore.reload(SolrCore.java:744)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1690)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3134)
        at app//org.apache.solr.core.SolrCore$$Lambda$729/0x000000014d2695b8.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2499)
        at app//org.apache.solr.cloud.ZkController$$Lambda$849/0x000000014daca0b0.run(Unknown Source)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   3) Thread[id=14611, name=searcherExecutor-9501-thread-1, state=WAITING, group=TGRP-TestBulkSchemaConcurrent]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   4) Thread[id=14546, name=ZKEventListenerThread, state=RUNNABLE, group=TGRP-TestBulkSchemaConcurrent]
        at app//org.apache.logging.log4j.core.impl.ThrowableProxyHelper.toExtendedStackTrace(ThrowableProxyHelper.java:98)
        at app//org.apache.logging.log4j.core.impl.ThrowableProxy.<init>(ThrowableProxy.java:112)
        at app//org.apache.logging.log4j.core.impl.ThrowableProxy.<init>(ThrowableProxy.java:96)
        at app//org.apache.logging.log4j.core.impl.Log4jLogEvent.getThrownProxy(Log4jLogEvent.java:629)
        at app//org.apache.logging.log4j.core.pattern.ExtendedThrowablePatternConverter.format(ExtendedThrowablePatternConverter.java:63)
        at app//org.apache.logging.log4j.core.layout.PatternLayout$NoFormatPatternSerializer.toSerializable(PatternLayout.java:343)
        at app//org.apache.logging.log4j.core.layout.PatternLayout.toText(PatternLayout.java:241)
        at app//org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:226)
        at app//org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:60)
        at app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:197)
        at app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:190)
        at app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:181)
        at app//org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:161)
        at app//org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:134)
        at app//org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:125)
        at app//org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:89)
        at app//org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:542)
        at app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:500)
        at app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:483)
        at app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:417)
        at app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82)
        at app//org.apache.logging.log4j.core.Logger.log(Logger.java:161)
        at app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205)
        at app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159)
        at app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142)
        at app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2017)
        at app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1983)
        at app//org.apache.logging.slf4j.Log4jLogger.error(Log4jLogger.java:319)
        at app//org.apache.solr.update.SolrIndexWriter.close(SolrIndexWriter.java:303)
        at app//org.apache.solr.update.DirectUpdateHandler2.closeWriter(DirectUpdateHandler2.java:895)
        at app//org.apache.solr.update.DefaultSolrCoreState.closeIndexWriter(DefaultSolrCoreState.java:96)
        at app//org.apache.solr.update.DefaultSolrCoreState.close(DefaultSolrCoreState.java:393)
        at app//org.apache.solr.update.SolrCoreState.decrefSolrCoreState(SolrCoreState.java:112)
        at app//org.apache.solr.core.SolrCore.doClose(SolrCore.java:1639)
        at app//org.apache.solr.core.SolrCore.close(SolrCore.java:1582)
        at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1100)
        at app//org.apache.solr.core.SolrCore.reload(SolrCore.java:744)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1690)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3134)
        at app//org.apache.solr.core.SolrCore$$Lambda$729/0x000000014d2695b8.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2499)
        at app//org.apache.solr.cloud.ZkController$$Lambda$849/0x000000014daca0b0.run(Unknown Source)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   5) Thread[id=14544, name=ZKEventListenerThread, state=RUNNABLE, group=TGRP-TestBulkSchemaConcurrent]
        at java.base@11.0.13/java.lang.StackTraceElement.initStackTraceElements(Native Method)
        at java.base@11.0.13/java.lang.StackTraceElement.of(StackTraceElement.java:526)
        at java.base@11.0.13/java.lang.Throwable.getOurStackTrace(Throwable.java:828)
        at java.base@11.0.13/java.lang.Throwable.printStackTrace(Throwable.java:660)
        at java.base@11.0.13/java.lang.Throwable.printStackTrace(Throwable.java:725)
        at app//org.apache.logging.log4j.core.pattern.ThrowablePatternConverter.formatOption(ThrowablePatternConverter.java:195)
        at app//org.apache.logging.log4j.core.pattern.ThrowablePatternConverter.format(ThrowablePatternConverter.java:137)
        at app//org.apache.logging.log4j.core.pattern.PatternFormatter.format(PatternFormatter.java:44)
        at app//org.apache.logging.log4j.core.pattern.VariablesNotEmptyReplacementConverter.format(VariablesNotEmptyReplacementConverter.java:83)
        at app//org.apache.logging.log4j.core.pattern.PatternFormatter.format(PatternFormatter.java:44)
        at app//org.apache.logging.log4j.core.pattern.MaxLengthConverter.format(MaxLengthConverter.java:92)
        at app//org.apache.logging.log4j.core.layout.PatternLayout$NoFormatPatternSerializer.toSerializable(PatternLayout.java:343)
        at app//org.apache.logging.log4j.core.layout.PatternLayout.toText(PatternLayout.java:241)
        at app//org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:226)
        at app//org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:60)
        at app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:197)
        at app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:190)
        at app//org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:181)
        at app//org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:161)
        at app//org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:134)
        at app//org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:125)
        at app//org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:89)
        at app//org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:542)
        at app//org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:500)
        at app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:483)
        at app//org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:417)
        at app//org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:82)
        at app//org.apache.logging.log4j.core.Logger.log(Logger.java:161)
        at app//org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2205)
        at app//org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2159)
        at app//org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2142)
        at app//org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2017)
        at app//org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1983)
        at app//org.apache.logging.slf4j.Log4jLogger.error(Log4jLogger.java:319)
        at app//org.apache.solr.update.SolrIndexWriter.close(SolrIndexWriter.java:303)
        at app//org.apache.solr.update.DirectUpdateHandler2.closeWriter(DirectUpdateHandler2.java:895)
        at app//org.apache.solr.update.DefaultSolrCoreState.closeIndexWriter(DefaultSolrCoreState.java:96)
        at app//org.apache.solr.update.DefaultSolrCoreState.close(DefaultSolrCoreState.java:393)
        at app//org.apache.solr.update.SolrCoreState.decrefSolrCoreState(SolrCoreState.java:112)
        at app//org.apache.solr.core.SolrCore.doClose(SolrCore.java:1639)
        at app//org.apache.solr.core.SolrCore.close(SolrCore.java:1582)
        at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1100)
        at app//org.apache.solr.core.SolrCore.reload(SolrCore.java:744)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1690)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3134)
        at app//org.apache.solr.core.SolrCore$$Lambda$729/0x000000014d2695b8.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2499)
        at app//org.apache.solr.cloud.ZkController$$Lambda$849/0x000000014daca0b0.run(Unknown Source)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   6) Thread[id=14613, name=searcherExecutor-9509-thread-1, state=WAITING, group=TGRP-TestBulkSchemaConcurrent]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   7) Thread[id=14614, name=searcherExecutor-9503-thread-1, state=WAITING, group=TGRP-TestBulkSchemaConcurrent]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   8) Thread[id=14603, name=searcherExecutor-9505-thread-1, state=WAITING, group=TGRP-TestBulkSchemaConcurrent]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
	at __randomizedtesting.SeedInfo.seed([44BB1B812E2AFB0]:0)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
java.security.AccessControlException: access denied ("java.io.FilePermission" "/Users/jenkins/workspace/Solr-main-MacOSX/solr/core/build/resources/test/solr/filestore" "write")

Stack Trace:
java.security.AccessControlException: access denied ("java.io.FilePermission" "/Users/jenkins/workspace/Solr-main-MacOSX/solr/core/build/resources/test/solr/filestore" "write")
	at __randomizedtesting.SeedInfo.seed([44BB1B812E2AFB0]:0)
	at java.base/java.security.AccessControlContext.checkPermission(AccessControlContext.java:472)
	at java.base/java.security.AccessController.checkPermission(AccessController.java:897)
	at java.base/java.lang.SecurityManager.checkPermission(SecurityManager.java:322)
	at java.base/java.lang.SecurityManager.checkWrite(SecurityManager.java:752)
	at java.base/sun.nio.fs.UnixPath.checkWrite(UnixPath.java:824)
	at java.base/sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:381)
	at java.base/java.nio.file.Files.createDirectory(Files.java:690)
	at java.base/java.nio.file.Files.createAndCheckIsDirectory(Files.java:797)
	at java.base/java.nio.file.Files.createDirectories(Files.java:743)
	at org.apache.solr.filestore.DistribPackageStore.getPackageStoreDirPath(DistribPackageStore.java:584)
	at org.apache.solr.filestore.DistribPackageStore.ensurePackageStoreDir(DistribPackageStore.java:567)
	at org.apache.solr.filestore.DistribPackageStore.<init>(DistribPackageStore.java:83)
	at org.apache.solr.filestore.PackageStoreAPI.<init>(PackageStoreAPI.java:76)
	at org.apache.solr.core.CoreContainer.load(CoreContainer.java:725)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:184)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:156)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:162)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:112)
	at org.apache.solr.SolrTestCaseJ4.createCore(SolrTestCaseJ4.java:813)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:803)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:650)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:639)
	at org.apache.solr.cloud.AbstractZkTestCase.azt_beforeClass(AbstractZkTestCase.java:68)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:882)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:829)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 8 threads leaked from SUITE scope at org.apache.solr.rest.TestManagedResourceStorage: 
   1) Thread[id=27394, name=SUITE-TestManagedResourceStorage-seed#[44BB1B812E2AFB0]-worker-EventThread, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:551)
   2) Thread[id=27398, name=zkCallback-17338-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   3) Thread[id=27386, name=httpShardExecutor-17332-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$262/0x000000014a403c58.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$218/0x000000014a9765b8.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   4) Thread[id=27396, name=OverseerStateUpdate-72080353192902659-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:294)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   5) Thread[id=27395, name=zkConnectionManagerCallback-17339-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   6) Thread[id=27397, name=OverseerCollectionConfigSetProcessor-72080353192902659-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:271)
        at app//org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:258)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   7) Thread[id=27389, name=h2sc-17334-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$262/0x000000014a403c58.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$218/0x000000014a9765b8.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   8) Thread[id=27393, name=SUITE-TestManagedResourceStorage-seed#[44BB1B812E2AFB0]-worker-SendThread(127.0.0.1:50869), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:214)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1380)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanAndNotifyState(ClientCnxn.java:1321)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1299)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 8 threads leaked from SUITE scope at org.apache.solr.rest.TestManagedResourceStorage: 
   1) Thread[id=27394, name=SUITE-TestManagedResourceStorage-seed#[44BB1B812E2AFB0]-worker-EventThread, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:551)
   2) Thread[id=27398, name=zkCallback-17338-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   3) Thread[id=27386, name=httpShardExecutor-17332-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$262/0x000000014a403c58.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$218/0x000000014a9765b8.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   4) Thread[id=27396, name=OverseerStateUpdate-72080353192902659-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:294)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   5) Thread[id=27395, name=zkConnectionManagerCallback-17339-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   6) Thread[id=27397, name=OverseerCollectionConfigSetProcessor-72080353192902659-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:271)
        at app//org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:258)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   7) Thread[id=27389, name=h2sc-17334-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$262/0x000000014a403c58.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$218/0x000000014a9765b8.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   8) Thread[id=27393, name=SUITE-TestManagedResourceStorage-seed#[44BB1B812E2AFB0]-worker-SendThread(127.0.0.1:50869), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:214)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1380)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanAndNotifyState(ClientCnxn.java:1321)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1299)
	at __randomizedtesting.SeedInfo.seed([44BB1B812E2AFB0]:0)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=27398, name=zkCallback-17338-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   2) Thread[id=27386, name=httpShardExecutor-17332-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$262/0x000000014a403c58.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$218/0x000000014a9765b8.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   3) Thread[id=27395, name=zkConnectionManagerCallback-17339-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   4) Thread[id=27389, name=h2sc-17334-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$262/0x000000014a403c58.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$218/0x000000014a9765b8.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   5) Thread[id=27393, name=SUITE-TestManagedResourceStorage-seed#[44BB1B812E2AFB0]-worker-SendThread(127.0.0.1:50869), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:362)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1197)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=27398, name=zkCallback-17338-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   2) Thread[id=27386, name=httpShardExecutor-17332-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$262/0x000000014a403c58.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$218/0x000000014a9765b8.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   3) Thread[id=27395, name=zkConnectionManagerCallback-17339-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   4) Thread[id=27389, name=h2sc-17334-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$262/0x000000014a403c58.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$218/0x000000014a9765b8.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   5) Thread[id=27393, name=SUITE-TestManagedResourceStorage-seed#[44BB1B812E2AFB0]-worker-SendThread(127.0.0.1:50869), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:362)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1197)
	at __randomizedtesting.SeedInfo.seed([44BB1B812E2AFB0]:0)

[JENKINS] Solr-main-MacOSX (64bit/jdk-11.0.13) - Build # 660 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/660/
Java: 64bit/jdk-11.0.13 -XX:-UseCompressedOops -XX:+UseSerialGC

5 tests failed.
FAILED:  org.apache.solr.cloud.MultiThreadedOCPTest.test

Error Message:
java.lang.AssertionError: Long running task finished too early, can't test expected null, but was:<1645195866626>

Stack Trace:
java.lang.AssertionError: Long running task finished too early, can't test expected null, but was:<1645195866626>
	at __randomizedtesting.SeedInfo.seed([A33BD0BB51656411:2B6FEF61FF9909E9]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.junit.Assert.failNotNull(Assert.java:756)
	at org.junit.Assert.assertNull(Assert.java:738)
	at org.apache.solr.cloud.MultiThreadedOCPTest.testFillWorkQueue(MultiThreadedOCPTest.java:87)
	at org.apache.solr.cloud.MultiThreadedOCPTest.test(MultiThreadedOCPTest.java:61)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1081)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1052)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:829)


FAILED:  org.apache.solr.schema.TestBulkSchemaConcurrent.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 10 threads leaked from SUITE scope at org.apache.solr.schema.TestBulkSchemaConcurrent: 
   1) Thread[id=8856, name=ZKEventListenerThread, state=RUNNABLE, group=TGRP-TestBulkSchemaConcurrent]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:885)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireShared(AbstractQueuedSynchronizer.java:1009)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireShared(AbstractQueuedSynchronizer.java:1324)
        at java.base@11.0.13/java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.lock(ReentrantReadWriteLock.java:738)
        at java.management@11.0.13/com.sun.jmx.mbeanserver.Repository.retrieve(Repository.java:484)
        at java.management@11.0.13/com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.getMBean(DefaultMBeanServerInterceptor.java:1077)
        at java.management@11.0.13/com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.getAttribute(DefaultMBeanServerInterceptor.java:637)
        at java.management@11.0.13/com.sun.jmx.mbeanserver.JmxMBeanServer.getAttribute(JmxMBeanServer.java:678)
        at java.management@11.0.13/javax.management.AttributeValueExp.getAttribute(AttributeValueExp.java:162)
        at java.management@11.0.13/javax.management.AttributeValueExp.apply(AttributeValueExp.java:104)
        at java.management@11.0.13/javax.management.BinaryRelQueryExp.apply(BinaryRelQueryExp.java:109)
        at java.management@11.0.13/com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.filterListOfObjectInstances(DefaultMBeanServerInterceptor.java:1599)
        at java.management@11.0.13/com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.queryMBeans(DefaultMBeanServerInterceptor.java:495)
        at java.management@11.0.13/com.sun.jmx.mbeanserver.JmxMBeanServer.queryMBeans(JmxMBeanServer.java:592)
        at app//org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter$JmxListener.unregisterMBean(JmxMetricsReporter.java:555)
        at app//org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter$JmxListener.unregisterAll(JmxMetricsReporter.java:709)
        at app//org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter.close(JmxMetricsReporter.java:759)
        at app//org.apache.solr.metrics.reporters.SolrJmxReporter.close(SolrJmxReporter.java:129)
        at app//org.apache.solr.metrics.SolrMetricManager.lambda$closeReporters$5(SolrMetricManager.java:1157)
        at app//org.apache.solr.metrics.SolrMetricManager$$Lambda$589/0x000000013fc37108.accept(Unknown Source)
        at java.base@11.0.13/java.lang.Iterable.forEach(Iterable.java:75)
        at app//org.apache.solr.metrics.SolrMetricManager.closeReporters(SolrMetricManager.java:1151)
        at app//org.apache.solr.metrics.SolrCoreMetricManager.close(SolrCoreMetricManager.java:145)
        at app//org.apache.solr.core.SolrCore.doClose(SolrCore.java:1596)
        at app//org.apache.solr.core.SolrCore.close(SolrCore.java:1582)
        at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1100)
        at app//org.apache.solr.core.SolrCore.reload(SolrCore.java:744)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1690)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3134)
        at app//org.apache.solr.core.SolrCore$$Lambda$844/0x000000014160f9b8.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2499)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1638/0x0000000142b884b0.run(Unknown Source)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   2) Thread[id=8857, name=ZKEventListenerThread, state=RUNNABLE, group=TGRP-TestBulkSchemaConcurrent]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:885)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireShared(AbstractQueuedSynchronizer.java:1009)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireShared(AbstractQueuedSynchronizer.java:1324)
        at java.base@11.0.13/java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.lock(ReentrantReadWriteLock.java:738)
        at java.management@11.0.13/com.sun.jmx.mbeanserver.Repository.retrieve(Repository.java:484)
        at java.management@11.0.13/com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.getMBean(DefaultMBeanServerInterceptor.java:1077)
        at java.management@11.0.13/com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.exclusiveUnregisterMBean(DefaultMBeanServerInterceptor.java:423)
        at java.management@11.0.13/com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.unregisterMBean(DefaultMBeanServerInterceptor.java:411)
        at java.management@11.0.13/com.sun.jmx.mbeanserver.JmxMBeanServer.unregisterMBean(JmxMBeanServer.java:546)
        at app//org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter$JmxListener.unregisterMBean(JmxMetricsReporter.java:560)
        at app//org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter$JmxListener.unregisterAll(JmxMetricsReporter.java:709)
        at app//org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter.close(JmxMetricsReporter.java:759)
        at app//org.apache.solr.metrics.reporters.SolrJmxReporter.close(SolrJmxReporter.java:129)
        at app//org.apache.solr.metrics.SolrMetricManager.lambda$closeReporters$5(SolrMetricManager.java:1157)
        at app//org.apache.solr.metrics.SolrMetricManager$$Lambda$589/0x000000013fc37108.accept(Unknown Source)
        at java.base@11.0.13/java.lang.Iterable.forEach(Iterable.java:75)
        at app//org.apache.solr.metrics.SolrMetricManager.closeReporters(SolrMetricManager.java:1151)
        at app//org.apache.solr.metrics.SolrCoreMetricManager.close(SolrCoreMetricManager.java:145)
        at app//org.apache.solr.core.SolrCore.doClose(SolrCore.java:1596)
        at app//org.apache.solr.core.SolrCore.close(SolrCore.java:1582)
        at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1100)
        at app//org.apache.solr.core.SolrCore.reload(SolrCore.java:744)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1690)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3134)
        at app//org.apache.solr.core.SolrCore$$Lambda$844/0x000000014160f9b8.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2499)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1638/0x0000000142b884b0.run(Unknown Source)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   3) Thread[id=8930, name=searcherExecutor-6674-thread-1, state=WAITING, group=TGRP-TestBulkSchemaConcurrent]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   4) Thread[id=8928, name=searcherExecutor-6670-thread-1, state=WAITING, group=TGRP-TestBulkSchemaConcurrent]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   5) Thread[id=8932, name=searcherExecutor-6672-thread-1, state=WAITING, group=TGRP-TestBulkSchemaConcurrent]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   6) Thread[id=8858, name=ZKEventListenerThread, state=RUNNABLE, group=TGRP-TestBulkSchemaConcurrent]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:885)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireShared(AbstractQueuedSynchronizer.java:1009)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireShared(AbstractQueuedSynchronizer.java:1324)
        at java.base@11.0.13/java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.lock(ReentrantReadWriteLock.java:738)
        at java.management@11.0.13/com.sun.jmx.mbeanserver.Repository.query(Repository.java:525)
        at java.management@11.0.13/com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.queryMBeansImpl(DefaultMBeanServerInterceptor.java:507)
        at java.management@11.0.13/com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.queryMBeans(DefaultMBeanServerInterceptor.java:476)
        at java.management@11.0.13/com.sun.jmx.mbeanserver.JmxMBeanServer.queryMBeans(JmxMBeanServer.java:592)
        at app//org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter$JmxListener.unregisterMBean(JmxMetricsReporter.java:555)
        at app//org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter$JmxListener.unregisterAll(JmxMetricsReporter.java:709)
        at app//org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter.close(JmxMetricsReporter.java:759)
        at app//org.apache.solr.metrics.reporters.SolrJmxReporter.close(SolrJmxReporter.java:129)
        at app//org.apache.solr.metrics.SolrMetricManager.lambda$closeReporters$5(SolrMetricManager.java:1157)
        at app//org.apache.solr.metrics.SolrMetricManager$$Lambda$589/0x000000013fc37108.accept(Unknown Source)
        at java.base@11.0.13/java.lang.Iterable.forEach(Iterable.java:75)
        at app//org.apache.solr.metrics.SolrMetricManager.closeReporters(SolrMetricManager.java:1151)
        at app//org.apache.solr.metrics.SolrCoreMetricManager.close(SolrCoreMetricManager.java:145)
        at app//org.apache.solr.core.SolrCore.doClose(SolrCore.java:1596)
        at app//org.apache.solr.core.SolrCore.close(SolrCore.java:1582)
        at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1100)
        at app//org.apache.solr.core.SolrCore.reload(SolrCore.java:744)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1690)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3134)
        at app//org.apache.solr.core.SolrCore$$Lambda$844/0x000000014160f9b8.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2499)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1638/0x0000000142b884b0.run(Unknown Source)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   7) Thread[id=8931, name=searcherExecutor-6666-thread-1, state=WAITING, group=TGRP-TestBulkSchemaConcurrent]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   8) Thread[id=8859, name=ZKEventListenerThread, state=WAITING, group=TGRP-TestBulkSchemaConcurrent]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:885)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireShared(AbstractQueuedSynchronizer.java:1009)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireShared(AbstractQueuedSynchronizer.java:1324)
        at java.base@11.0.13/java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.lock(ReentrantReadWriteLock.java:738)
        at java.management@11.0.13/com.sun.jmx.mbeanserver.Repository.query(Repository.java:525)
        at java.management@11.0.13/com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.queryMBeansImpl(DefaultMBeanServerInterceptor.java:507)
        at java.management@11.0.13/com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.queryMBeans(DefaultMBeanServerInterceptor.java:476)
        at java.management@11.0.13/com.sun.jmx.mbeanserver.JmxMBeanServer.queryMBeans(JmxMBeanServer.java:592)
        at app//org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter$JmxListener.unregisterMBean(JmxMetricsReporter.java:555)
        at app//org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter$JmxListener.unregisterAll(JmxMetricsReporter.java:709)
        at app//org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter.close(JmxMetricsReporter.java:759)
        at app//org.apache.solr.metrics.reporters.SolrJmxReporter.close(SolrJmxReporter.java:129)
        at app//org.apache.solr.metrics.SolrMetricManager.lambda$closeReporters$5(SolrMetricManager.java:1157)
        at app//org.apache.solr.metrics.SolrMetricManager$$Lambda$589/0x000000013fc37108.accept(Unknown Source)
        at java.base@11.0.13/java.lang.Iterable.forEach(Iterable.java:75)
        at app//org.apache.solr.metrics.SolrMetricManager.closeReporters(SolrMetricManager.java:1151)
        at app//org.apache.solr.metrics.SolrCoreMetricManager.close(SolrCoreMetricManager.java:145)
        at app//org.apache.solr.core.SolrCore.doClose(SolrCore.java:1596)
        at app//org.apache.solr.core.SolrCore.close(SolrCore.java:1582)
        at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1100)
        at app//org.apache.solr.core.SolrCore.reload(SolrCore.java:744)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1690)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3134)
        at app//org.apache.solr.core.SolrCore$$Lambda$844/0x000000014160f9b8.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2499)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1638/0x0000000142b884b0.run(Unknown Source)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   9) Thread[id=8929, name=searcherExecutor-6668-thread-1, state=WAITING, group=TGRP-TestBulkSchemaConcurrent]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  10) Thread[id=8850, name=ZKEventListenerThread, state=WAITING, group=TGRP-TestBulkSchemaConcurrent]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:885)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireShared(AbstractQueuedSynchronizer.java:1009)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireShared(AbstractQueuedSynchronizer.java:1324)
        at java.base@11.0.13/java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.lock(ReentrantReadWriteLock.java:738)
        at java.management@11.0.13/com.sun.jmx.mbeanserver.Repository.query(Repository.java:525)
        at java.management@11.0.13/com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.queryMBeansImpl(DefaultMBeanServerInterceptor.java:507)
        at java.management@11.0.13/com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.queryMBeans(DefaultMBeanServerInterceptor.java:476)
        at java.management@11.0.13/com.sun.jmx.mbeanserver.JmxMBeanServer.queryMBeans(JmxMBeanServer.java:592)
        at app//org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter$JmxListener.unregisterMBean(JmxMetricsReporter.java:555)
        at app//org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter$JmxListener.unregisterAll(JmxMetricsReporter.java:709)
        at app//org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter.close(JmxMetricsReporter.java:759)
        at app//org.apache.solr.metrics.reporters.SolrJmxReporter.close(SolrJmxReporter.java:129)
        at app//org.apache.solr.metrics.SolrMetricManager.lambda$closeReporters$5(SolrMetricManager.java:1157)
        at app//org.apache.solr.metrics.SolrMetricManager$$Lambda$589/0x000000013fc37108.accept(Unknown Source)
        at java.base@11.0.13/java.lang.Iterable.forEach(Iterable.java:75)
        at app//org.apache.solr.metrics.SolrMetricManager.closeReporters(SolrMetricManager.java:1151)
        at app//org.apache.solr.metrics.SolrCoreMetricManager.close(SolrCoreMetricManager.java:145)
        at app//org.apache.solr.core.SolrCore.doClose(SolrCore.java:1596)
        at app//org.apache.solr.core.SolrCore.close(SolrCore.java:1582)
        at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1100)
        at app//org.apache.solr.core.SolrCore.reload(SolrCore.java:744)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1690)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3134)
        at app//org.apache.solr.core.SolrCore$$Lambda$844/0x000000014160f9b8.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2499)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1638/0x0000000142b884b0.run(Unknown Source)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 10 threads leaked from SUITE scope at org.apache.solr.schema.TestBulkSchemaConcurrent: 
   1) Thread[id=8856, name=ZKEventListenerThread, state=RUNNABLE, group=TGRP-TestBulkSchemaConcurrent]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:885)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireShared(AbstractQueuedSynchronizer.java:1009)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireShared(AbstractQueuedSynchronizer.java:1324)
        at java.base@11.0.13/java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.lock(ReentrantReadWriteLock.java:738)
        at java.management@11.0.13/com.sun.jmx.mbeanserver.Repository.retrieve(Repository.java:484)
        at java.management@11.0.13/com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.getMBean(DefaultMBeanServerInterceptor.java:1077)
        at java.management@11.0.13/com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.getAttribute(DefaultMBeanServerInterceptor.java:637)
        at java.management@11.0.13/com.sun.jmx.mbeanserver.JmxMBeanServer.getAttribute(JmxMBeanServer.java:678)
        at java.management@11.0.13/javax.management.AttributeValueExp.getAttribute(AttributeValueExp.java:162)
        at java.management@11.0.13/javax.management.AttributeValueExp.apply(AttributeValueExp.java:104)
        at java.management@11.0.13/javax.management.BinaryRelQueryExp.apply(BinaryRelQueryExp.java:109)
        at java.management@11.0.13/com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.filterListOfObjectInstances(DefaultMBeanServerInterceptor.java:1599)
        at java.management@11.0.13/com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.queryMBeans(DefaultMBeanServerInterceptor.java:495)
        at java.management@11.0.13/com.sun.jmx.mbeanserver.JmxMBeanServer.queryMBeans(JmxMBeanServer.java:592)
        at app//org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter$JmxListener.unregisterMBean(JmxMetricsReporter.java:555)
        at app//org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter$JmxListener.unregisterAll(JmxMetricsReporter.java:709)
        at app//org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter.close(JmxMetricsReporter.java:759)
        at app//org.apache.solr.metrics.reporters.SolrJmxReporter.close(SolrJmxReporter.java:129)
        at app//org.apache.solr.metrics.SolrMetricManager.lambda$closeReporters$5(SolrMetricManager.java:1157)
        at app//org.apache.solr.metrics.SolrMetricManager$$Lambda$589/0x000000013fc37108.accept(Unknown Source)
        at java.base@11.0.13/java.lang.Iterable.forEach(Iterable.java:75)
        at app//org.apache.solr.metrics.SolrMetricManager.closeReporters(SolrMetricManager.java:1151)
        at app//org.apache.solr.metrics.SolrCoreMetricManager.close(SolrCoreMetricManager.java:145)
        at app//org.apache.solr.core.SolrCore.doClose(SolrCore.java:1596)
        at app//org.apache.solr.core.SolrCore.close(SolrCore.java:1582)
        at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1100)
        at app//org.apache.solr.core.SolrCore.reload(SolrCore.java:744)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1690)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3134)
        at app//org.apache.solr.core.SolrCore$$Lambda$844/0x000000014160f9b8.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2499)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1638/0x0000000142b884b0.run(Unknown Source)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   2) Thread[id=8857, name=ZKEventListenerThread, state=RUNNABLE, group=TGRP-TestBulkSchemaConcurrent]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:885)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireShared(AbstractQueuedSynchronizer.java:1009)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireShared(AbstractQueuedSynchronizer.java:1324)
        at java.base@11.0.13/java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.lock(ReentrantReadWriteLock.java:738)
        at java.management@11.0.13/com.sun.jmx.mbeanserver.Repository.retrieve(Repository.java:484)
        at java.management@11.0.13/com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.getMBean(DefaultMBeanServerInterceptor.java:1077)
        at java.management@11.0.13/com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.exclusiveUnregisterMBean(DefaultMBeanServerInterceptor.java:423)
        at java.management@11.0.13/com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.unregisterMBean(DefaultMBeanServerInterceptor.java:411)
        at java.management@11.0.13/com.sun.jmx.mbeanserver.JmxMBeanServer.unregisterMBean(JmxMBeanServer.java:546)
        at app//org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter$JmxListener.unregisterMBean(JmxMetricsReporter.java:560)
        at app//org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter$JmxListener.unregisterAll(JmxMetricsReporter.java:709)
        at app//org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter.close(JmxMetricsReporter.java:759)
        at app//org.apache.solr.metrics.reporters.SolrJmxReporter.close(SolrJmxReporter.java:129)
        at app//org.apache.solr.metrics.SolrMetricManager.lambda$closeReporters$5(SolrMetricManager.java:1157)
        at app//org.apache.solr.metrics.SolrMetricManager$$Lambda$589/0x000000013fc37108.accept(Unknown Source)
        at java.base@11.0.13/java.lang.Iterable.forEach(Iterable.java:75)
        at app//org.apache.solr.metrics.SolrMetricManager.closeReporters(SolrMetricManager.java:1151)
        at app//org.apache.solr.metrics.SolrCoreMetricManager.close(SolrCoreMetricManager.java:145)
        at app//org.apache.solr.core.SolrCore.doClose(SolrCore.java:1596)
        at app//org.apache.solr.core.SolrCore.close(SolrCore.java:1582)
        at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1100)
        at app//org.apache.solr.core.SolrCore.reload(SolrCore.java:744)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1690)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3134)
        at app//org.apache.solr.core.SolrCore$$Lambda$844/0x000000014160f9b8.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2499)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1638/0x0000000142b884b0.run(Unknown Source)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   3) Thread[id=8930, name=searcherExecutor-6674-thread-1, state=WAITING, group=TGRP-TestBulkSchemaConcurrent]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   4) Thread[id=8928, name=searcherExecutor-6670-thread-1, state=WAITING, group=TGRP-TestBulkSchemaConcurrent]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   5) Thread[id=8932, name=searcherExecutor-6672-thread-1, state=WAITING, group=TGRP-TestBulkSchemaConcurrent]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   6) Thread[id=8858, name=ZKEventListenerThread, state=RUNNABLE, group=TGRP-TestBulkSchemaConcurrent]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:885)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireShared(AbstractQueuedSynchronizer.java:1009)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireShared(AbstractQueuedSynchronizer.java:1324)
        at java.base@11.0.13/java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.lock(ReentrantReadWriteLock.java:738)
        at java.management@11.0.13/com.sun.jmx.mbeanserver.Repository.query(Repository.java:525)
        at java.management@11.0.13/com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.queryMBeansImpl(DefaultMBeanServerInterceptor.java:507)
        at java.management@11.0.13/com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.queryMBeans(DefaultMBeanServerInterceptor.java:476)
        at java.management@11.0.13/com.sun.jmx.mbeanserver.JmxMBeanServer.queryMBeans(JmxMBeanServer.java:592)
        at app//org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter$JmxListener.unregisterMBean(JmxMetricsReporter.java:555)
        at app//org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter$JmxListener.unregisterAll(JmxMetricsReporter.java:709)
        at app//org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter.close(JmxMetricsReporter.java:759)
        at app//org.apache.solr.metrics.reporters.SolrJmxReporter.close(SolrJmxReporter.java:129)
        at app//org.apache.solr.metrics.SolrMetricManager.lambda$closeReporters$5(SolrMetricManager.java:1157)
        at app//org.apache.solr.metrics.SolrMetricManager$$Lambda$589/0x000000013fc37108.accept(Unknown Source)
        at java.base@11.0.13/java.lang.Iterable.forEach(Iterable.java:75)
        at app//org.apache.solr.metrics.SolrMetricManager.closeReporters(SolrMetricManager.java:1151)
        at app//org.apache.solr.metrics.SolrCoreMetricManager.close(SolrCoreMetricManager.java:145)
        at app//org.apache.solr.core.SolrCore.doClose(SolrCore.java:1596)
        at app//org.apache.solr.core.SolrCore.close(SolrCore.java:1582)
        at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1100)
        at app//org.apache.solr.core.SolrCore.reload(SolrCore.java:744)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1690)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3134)
        at app//org.apache.solr.core.SolrCore$$Lambda$844/0x000000014160f9b8.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2499)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1638/0x0000000142b884b0.run(Unknown Source)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   7) Thread[id=8931, name=searcherExecutor-6666-thread-1, state=WAITING, group=TGRP-TestBulkSchemaConcurrent]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   8) Thread[id=8859, name=ZKEventListenerThread, state=WAITING, group=TGRP-TestBulkSchemaConcurrent]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:885)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireShared(AbstractQueuedSynchronizer.java:1009)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireShared(AbstractQueuedSynchronizer.java:1324)
        at java.base@11.0.13/java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.lock(ReentrantReadWriteLock.java:738)
        at java.management@11.0.13/com.sun.jmx.mbeanserver.Repository.query(Repository.java:525)
        at java.management@11.0.13/com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.queryMBeansImpl(DefaultMBeanServerInterceptor.java:507)
        at java.management@11.0.13/com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.queryMBeans(DefaultMBeanServerInterceptor.java:476)
        at java.management@11.0.13/com.sun.jmx.mbeanserver.JmxMBeanServer.queryMBeans(JmxMBeanServer.java:592)
        at app//org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter$JmxListener.unregisterMBean(JmxMetricsReporter.java:555)
        at app//org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter$JmxListener.unregisterAll(JmxMetricsReporter.java:709)
        at app//org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter.close(JmxMetricsReporter.java:759)
        at app//org.apache.solr.metrics.reporters.SolrJmxReporter.close(SolrJmxReporter.java:129)
        at app//org.apache.solr.metrics.SolrMetricManager.lambda$closeReporters$5(SolrMetricManager.java:1157)
        at app//org.apache.solr.metrics.SolrMetricManager$$Lambda$589/0x000000013fc37108.accept(Unknown Source)
        at java.base@11.0.13/java.lang.Iterable.forEach(Iterable.java:75)
        at app//org.apache.solr.metrics.SolrMetricManager.closeReporters(SolrMetricManager.java:1151)
        at app//org.apache.solr.metrics.SolrCoreMetricManager.close(SolrCoreMetricManager.java:145)
        at app//org.apache.solr.core.SolrCore.doClose(SolrCore.java:1596)
        at app//org.apache.solr.core.SolrCore.close(SolrCore.java:1582)
        at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1100)
        at app//org.apache.solr.core.SolrCore.reload(SolrCore.java:744)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1690)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3134)
        at app//org.apache.solr.core.SolrCore$$Lambda$844/0x000000014160f9b8.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2499)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1638/0x0000000142b884b0.run(Unknown Source)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   9) Thread[id=8929, name=searcherExecutor-6668-thread-1, state=WAITING, group=TGRP-TestBulkSchemaConcurrent]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  10) Thread[id=8850, name=ZKEventListenerThread, state=WAITING, group=TGRP-TestBulkSchemaConcurrent]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:885)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireShared(AbstractQueuedSynchronizer.java:1009)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireShared(AbstractQueuedSynchronizer.java:1324)
        at java.base@11.0.13/java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.lock(ReentrantReadWriteLock.java:738)
        at java.management@11.0.13/com.sun.jmx.mbeanserver.Repository.query(Repository.java:525)
        at java.management@11.0.13/com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.queryMBeansImpl(DefaultMBeanServerInterceptor.java:507)
        at java.management@11.0.13/com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.queryMBeans(DefaultMBeanServerInterceptor.java:476)
        at java.management@11.0.13/com.sun.jmx.mbeanserver.JmxMBeanServer.queryMBeans(JmxMBeanServer.java:592)
        at app//org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter$JmxListener.unregisterMBean(JmxMetricsReporter.java:555)
        at app//org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter$JmxListener.unregisterAll(JmxMetricsReporter.java:709)
        at app//org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter.close(JmxMetricsReporter.java:759)
        at app//org.apache.solr.metrics.reporters.SolrJmxReporter.close(SolrJmxReporter.java:129)
        at app//org.apache.solr.metrics.SolrMetricManager.lambda$closeReporters$5(SolrMetricManager.java:1157)
        at app//org.apache.solr.metrics.SolrMetricManager$$Lambda$589/0x000000013fc37108.accept(Unknown Source)
        at java.base@11.0.13/java.lang.Iterable.forEach(Iterable.java:75)
        at app//org.apache.solr.metrics.SolrMetricManager.closeReporters(SolrMetricManager.java:1151)
        at app//org.apache.solr.metrics.SolrCoreMetricManager.close(SolrCoreMetricManager.java:145)
        at app//org.apache.solr.core.SolrCore.doClose(SolrCore.java:1596)
        at app//org.apache.solr.core.SolrCore.close(SolrCore.java:1582)
        at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1100)
        at app//org.apache.solr.core.SolrCore.reload(SolrCore.java:744)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1690)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3134)
        at app//org.apache.solr.core.SolrCore$$Lambda$844/0x000000014160f9b8.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2499)
        at app//org.apache.solr.cloud.ZkController$$Lambda$1638/0x0000000142b884b0.run(Unknown Source)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
	at __randomizedtesting.SeedInfo.seed([A33BD0BB51656411]:0)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
java.security.AccessControlException: access denied ("java.io.FilePermission" "/Users/jenkins/workspace/Solr-main-MacOSX/solr/core/build/resources/test/solr/filestore" "write")

Stack Trace:
java.security.AccessControlException: access denied ("java.io.FilePermission" "/Users/jenkins/workspace/Solr-main-MacOSX/solr/core/build/resources/test/solr/filestore" "write")
	at __randomizedtesting.SeedInfo.seed([A33BD0BB51656411]:0)
	at java.base/java.security.AccessControlContext.checkPermission(AccessControlContext.java:472)
	at java.base/java.security.AccessController.checkPermission(AccessController.java:897)
	at java.base/java.lang.SecurityManager.checkPermission(SecurityManager.java:322)
	at java.base/java.lang.SecurityManager.checkWrite(SecurityManager.java:752)
	at java.base/sun.nio.fs.UnixPath.checkWrite(UnixPath.java:824)
	at java.base/sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:381)
	at java.base/java.nio.file.Files.createDirectory(Files.java:690)
	at java.base/java.nio.file.Files.createAndCheckIsDirectory(Files.java:797)
	at java.base/java.nio.file.Files.createDirectories(Files.java:743)
	at org.apache.solr.filestore.DistribPackageStore.getPackageStoreDirPath(DistribPackageStore.java:584)
	at org.apache.solr.filestore.DistribPackageStore.ensurePackageStoreDir(DistribPackageStore.java:567)
	at org.apache.solr.filestore.DistribPackageStore.<init>(DistribPackageStore.java:83)
	at org.apache.solr.filestore.PackageStoreAPI.<init>(PackageStoreAPI.java:76)
	at org.apache.solr.core.CoreContainer.load(CoreContainer.java:725)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:184)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:156)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:162)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:112)
	at org.apache.solr.SolrTestCaseJ4.createCore(SolrTestCaseJ4.java:813)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:803)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:650)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:639)
	at org.apache.solr.cloud.AbstractZkTestCase.azt_beforeClass(AbstractZkTestCase.java:68)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:882)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:829)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 8 threads leaked from SUITE scope at org.apache.solr.rest.TestManagedResourceStorage: 
   1) Thread[id=20132, name=OverseerStateUpdate-72079404392841219-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:294)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   2) Thread[id=20134, name=zkCallback-13088-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   3) Thread[id=20131, name=zkConnectionManagerCallback-13089-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   4) Thread[id=20130, name=SUITE-TestManagedResourceStorage-seed#[A33BD0BB51656411]-worker-EventThread, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:551)
   5) Thread[id=20133, name=OverseerCollectionConfigSetProcessor-72079404392841219-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:271)
        at app//org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:258)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   6) Thread[id=20125, name=h2sc-13084-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$207/0x000000013e5f5458.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$246/0x000000013edbe9b8.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   7) Thread[id=20122, name=httpShardExecutor-13082-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$207/0x000000013e5f5458.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$246/0x000000013edbe9b8.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   8) Thread[id=20129, name=SUITE-TestManagedResourceStorage-seed#[A33BD0BB51656411]-worker-SendThread(127.0.0.1:64887), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:362)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1197)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 8 threads leaked from SUITE scope at org.apache.solr.rest.TestManagedResourceStorage: 
   1) Thread[id=20132, name=OverseerStateUpdate-72079404392841219-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:294)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   2) Thread[id=20134, name=zkCallback-13088-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   3) Thread[id=20131, name=zkConnectionManagerCallback-13089-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   4) Thread[id=20130, name=SUITE-TestManagedResourceStorage-seed#[A33BD0BB51656411]-worker-EventThread, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:551)
   5) Thread[id=20133, name=OverseerCollectionConfigSetProcessor-72079404392841219-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:271)
        at app//org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:258)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   6) Thread[id=20125, name=h2sc-13084-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$207/0x000000013e5f5458.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$246/0x000000013edbe9b8.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   7) Thread[id=20122, name=httpShardExecutor-13082-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$207/0x000000013e5f5458.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$246/0x000000013edbe9b8.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   8) Thread[id=20129, name=SUITE-TestManagedResourceStorage-seed#[A33BD0BB51656411]-worker-SendThread(127.0.0.1:64887), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:362)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1197)
	at __randomizedtesting.SeedInfo.seed([A33BD0BB51656411]:0)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=20134, name=zkCallback-13088-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   2) Thread[id=20131, name=zkConnectionManagerCallback-13089-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   3) Thread[id=20125, name=h2sc-13084-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$207/0x000000013e5f5458.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$246/0x000000013edbe9b8.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   4) Thread[id=20122, name=httpShardExecutor-13082-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$207/0x000000013e5f5458.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$246/0x000000013edbe9b8.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   5) Thread[id=20129, name=SUITE-TestManagedResourceStorage-seed#[A33BD0BB51656411]-worker-SendThread(127.0.0.1:64887), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:362)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1197)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=20134, name=zkCallback-13088-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   2) Thread[id=20131, name=zkConnectionManagerCallback-13089-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   3) Thread[id=20125, name=h2sc-13084-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$207/0x000000013e5f5458.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$246/0x000000013edbe9b8.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   4) Thread[id=20122, name=httpShardExecutor-13082-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$207/0x000000013e5f5458.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$246/0x000000013edbe9b8.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   5) Thread[id=20129, name=SUITE-TestManagedResourceStorage-seed#[A33BD0BB51656411]-worker-SendThread(127.0.0.1:64887), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:362)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1197)
	at __randomizedtesting.SeedInfo.seed([A33BD0BB51656411]:0)

[JENKINS] Solr-main-MacOSX (64bit/jdk-16.0.2) - Build # 659 - Still unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/659/
Java: 64bit/jdk-16.0.2 -XX:-UseCompressedOops -XX:+UseParallelGC

9 tests failed.
FAILED:  org.apache.solr.cloud.TestQueryingOnDownCollection.testQueryToDownCollectionShouldFailFast

Error Message:
junit.framework.AssertionFailedError: Request should fail after trying all replica nodes once

Stack Trace:
junit.framework.AssertionFailedError: Request should fail after trying all replica nodes once
	at __randomizedtesting.SeedInfo.seed([91F5FD1ED110E740:4228D4B476218E3C]:0)
	at org.apache.lucene.util.LuceneTestCase.expectThrows(LuceneTestCase.java:2874)
	at org.apache.solr.cloud.TestQueryingOnDownCollection.testQueryToDownCollectionShouldFailFast(TestQueryingOnDownCollection.java:92)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:78)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:567)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:831)


FAILED:  org.apache.solr.schema.PreAnalyzedFieldManagedSchemaCloudTest.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=12936, name=qtp549510336-12936, state=RUNNABLE, group=TGRP-PreAnalyzedFieldManagedSchemaCloudTest], registration stack trace below.

Stack Trace:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=12936, name=qtp549510336-12936, state=RUNNABLE, group=TGRP-PreAnalyzedFieldManagedSchemaCloudTest], registration stack trace below.
	at java.base@16.0.2/java.lang.Thread.getStackTrace(Thread.java:1596)
	at app//com.carrotsearch.randomizedtesting.RandomizedContext.closeAtEnd(RandomizedContext.java:157)
	at app//org.apache.lucene.util.LuceneTestCase.closeAfterSuite(LuceneTestCase.java:798)
	at app//org.apache.lucene.util.LuceneTestCase.wrapDirectory(LuceneTestCase.java:1504)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1368)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1350)
	at app//org.apache.solr.core.MockDirectoryFactory.create(MockDirectoryFactory.java:49)
	at app//org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:360)
	at app//org.apache.solr.core.SolrCore.initSnapshotMetaDataManager(SolrCore.java:582)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1026)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:961)
	at app//org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1423)
	at app//org.apache.solr.core.CoreContainer.create(CoreContainer.java:1297)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:345)
	at app//org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:423)
	at app//org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:197)
	at app//org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:214)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:866)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:830)
	at app//org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:542)
	at app//org.apache.solr.servlet.SolrDispatchFilter.dispatch(SolrDispatchFilter.java:218)
	at app//org.apache.solr.servlet.SolrDispatchFilter.lambda$doFilter$0(SolrDispatchFilter.java:189)
	at app//org.apache.solr.servlet.ServletUtils.traceHttpRequestExecution2(ServletUtils.java:232)
	at app//org.apache.solr.servlet.ServletUtils.rateLimitRequest(ServletUtils.java:208)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:187)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:171)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:189)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1434)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
	at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)
	at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1349)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
	at app//org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:713)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.server.Server.handle(Server.java:516)
	at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:400)
	at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:645)
	at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:392)
	at app//org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:352)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
	at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
Caused by: java.lang.AssertionError: Directory not closed: MockDirectoryWrapper(ByteBuffersDirectory@182c8359 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4edf7c5b)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.lucene.util.CloseableDirectory.close(CloseableDirectory.java:43)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:788)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:785)
	at com.carrotsearch.randomizedtesting.RandomizedContext.closeResources(RandomizedContext.java:225)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.afterAlways(RandomizedRunner.java:801)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	... 1 more


FAILED:  org.apache.solr.schema.PreAnalyzedFieldManagedSchemaCloudTest.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=12936, name=qtp549510336-12936, state=RUNNABLE, group=TGRP-PreAnalyzedFieldManagedSchemaCloudTest], registration stack trace below.

Stack Trace:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=12936, name=qtp549510336-12936, state=RUNNABLE, group=TGRP-PreAnalyzedFieldManagedSchemaCloudTest], registration stack trace below.
	at java.base@16.0.2/java.lang.Thread.getStackTrace(Thread.java:1596)
	at app//com.carrotsearch.randomizedtesting.RandomizedContext.closeAtEnd(RandomizedContext.java:157)
	at app//org.apache.lucene.util.LuceneTestCase.closeAfterSuite(LuceneTestCase.java:798)
	at app//org.apache.lucene.util.LuceneTestCase.wrapDirectory(LuceneTestCase.java:1504)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1368)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1350)
	at app//org.apache.solr.core.MockDirectoryFactory.create(MockDirectoryFactory.java:49)
	at app//org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:360)
	at app//org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:404)
	at app//org.apache.solr.core.SolrCore.initIndex(SolrCore.java:817)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1030)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:961)
	at app//org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1423)
	at app//org.apache.solr.core.CoreContainer.create(CoreContainer.java:1297)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:345)
	at app//org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:423)
	at app//org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:197)
	at app//org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:214)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:866)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:830)
	at app//org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:542)
	at app//org.apache.solr.servlet.SolrDispatchFilter.dispatch(SolrDispatchFilter.java:218)
	at app//org.apache.solr.servlet.SolrDispatchFilter.lambda$doFilter$0(SolrDispatchFilter.java:189)
	at app//org.apache.solr.servlet.ServletUtils.traceHttpRequestExecution2(ServletUtils.java:232)
	at app//org.apache.solr.servlet.ServletUtils.rateLimitRequest(ServletUtils.java:208)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:187)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:171)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:189)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1434)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
	at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)
	at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1349)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
	at app//org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:713)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.server.Server.handle(Server.java:516)
	at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:400)
	at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:645)
	at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:392)
	at app//org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:352)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
	at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
Caused by: java.lang.AssertionError: Directory not closed: MockDirectoryWrapper(ByteBuffersDirectory@3065272f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7ed6290)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.lucene.util.CloseableDirectory.close(CloseableDirectory.java:43)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:788)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:785)
	at com.carrotsearch.randomizedtesting.RandomizedContext.closeResources(RandomizedContext.java:225)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.afterAlways(RandomizedRunner.java:801)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	... 1 more


FAILED:  org.apache.solr.schema.PreAnalyzedFieldManagedSchemaCloudTest.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=12936, name=qtp549510336-12936, state=RUNNABLE, group=TGRP-PreAnalyzedFieldManagedSchemaCloudTest], registration stack trace below.

Stack Trace:
com.carrotsearch.randomizedtesting.ResourceDisposalError: Resource in scope SUITE failed to close. Resource was registered from thread Thread[id=12936, name=qtp549510336-12936, state=RUNNABLE, group=TGRP-PreAnalyzedFieldManagedSchemaCloudTest], registration stack trace below.
	at java.base@16.0.2/java.lang.Thread.getStackTrace(Thread.java:1596)
	at app//com.carrotsearch.randomizedtesting.RandomizedContext.closeAtEnd(RandomizedContext.java:157)
	at app//org.apache.lucene.util.LuceneTestCase.closeAfterSuite(LuceneTestCase.java:798)
	at app//org.apache.lucene.util.LuceneTestCase.wrapDirectory(LuceneTestCase.java:1504)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1368)
	at app//org.apache.lucene.util.LuceneTestCase.newDirectory(LuceneTestCase.java:1350)
	at app//org.apache.solr.core.MockDirectoryFactory.create(MockDirectoryFactory.java:49)
	at app//org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:360)
	at app//org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:98)
	at app//org.apache.solr.core.SolrCore.initIndex(SolrCore.java:849)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1030)
	at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:961)
	at app//org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1423)
	at app//org.apache.solr.core.CoreContainer.create(CoreContainer.java:1297)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
	at app//org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:345)
	at app//org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:423)
	at app//org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:197)
	at app//org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:214)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:866)
	at app//org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:830)
	at app//org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:542)
	at app//org.apache.solr.servlet.SolrDispatchFilter.dispatch(SolrDispatchFilter.java:218)
	at app//org.apache.solr.servlet.SolrDispatchFilter.lambda$doFilter$0(SolrDispatchFilter.java:189)
	at app//org.apache.solr.servlet.ServletUtils.traceHttpRequestExecution2(ServletUtils.java:232)
	at app//org.apache.solr.servlet.ServletUtils.rateLimitRequest(ServletUtils.java:208)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:187)
	at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:171)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:189)
	at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
	at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
	at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1434)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
	at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)
	at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
	at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1349)
	at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
	at app//org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:713)
	at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at app//org.eclipse.jetty.server.Server.handle(Server.java:516)
	at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:400)
	at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:645)
	at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:392)
	at app//org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:352)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
	at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
	at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
Caused by: java.lang.AssertionError: Directory not closed: MockDirectoryWrapper(ByteBuffersDirectory@1de2946a lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@39b3ec0e)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.lucene.util.CloseableDirectory.close(CloseableDirectory.java:43)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:788)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4$1.apply(RandomizedRunner.java:785)
	at com.carrotsearch.randomizedtesting.RandomizedContext.closeResources(RandomizedContext.java:225)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.afterAlways(RandomizedRunner.java:801)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	... 1 more


FAILED:  org.apache.solr.schema.PreAnalyzedFieldManagedSchemaCloudTest.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.schema.PreAnalyzedFieldManagedSchemaCloudTest: 
   1) Thread[id=13008, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-PreAnalyzedFieldManagedSchemaCloudTest]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:156)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:89)
        at app//org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:306)
        at app//org.apache.solr.rest.ManagedResourceStorage$ZooKeeperStorageIO.configure(ManagedResourceStorage.java:232)
        at app//org.apache.solr.rest.ManagedResourceStorage.newStorageIO(ManagedResourceStorage.java:144)
        at app//org.apache.solr.core.SolrCore.initRestManager(SolrCore.java:2968)
        at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1057)
        at app//org.apache.solr.core.SolrCore.reload(SolrCore.java:744)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1690)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3134)
        at app//org.apache.solr.core.SolrCore$$Lambda$724/0x0000000801295bf0.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2499)
        at app//org.apache.solr.cloud.ZkController$$Lambda$848/0x00000008012e7900.run(Unknown Source)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   2) Thread[id=13032, name=searcherExecutor-7468-thread-1-processing-127.0.0.1:56731_solr, state=WAITING, group=TGRP-PreAnalyzedFieldManagedSchemaCloudTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:714)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1046)
        at java.base@16.0.2/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:232)
        at app//org.apache.solr.core.SolrCore.lambda$new$2(SolrCore.java:1048)
        at app//org.apache.solr.core.SolrCore$$Lambda$698/0x000000080127e018.call(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.FutureTask.run(FutureTask.java:264)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$234/0x0000000800ed54e8.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.schema.PreAnalyzedFieldManagedSchemaCloudTest: 
   1) Thread[id=13008, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-PreAnalyzedFieldManagedSchemaCloudTest]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:156)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:89)
        at app//org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:306)
        at app//org.apache.solr.rest.ManagedResourceStorage$ZooKeeperStorageIO.configure(ManagedResourceStorage.java:232)
        at app//org.apache.solr.rest.ManagedResourceStorage.newStorageIO(ManagedResourceStorage.java:144)
        at app//org.apache.solr.core.SolrCore.initRestManager(SolrCore.java:2968)
        at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1057)
        at app//org.apache.solr.core.SolrCore.reload(SolrCore.java:744)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1690)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3134)
        at app//org.apache.solr.core.SolrCore$$Lambda$724/0x0000000801295bf0.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2499)
        at app//org.apache.solr.cloud.ZkController$$Lambda$848/0x00000008012e7900.run(Unknown Source)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   2) Thread[id=13032, name=searcherExecutor-7468-thread-1-processing-127.0.0.1:56731_solr, state=WAITING, group=TGRP-PreAnalyzedFieldManagedSchemaCloudTest]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:714)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1046)
        at java.base@16.0.2/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:232)
        at app//org.apache.solr.core.SolrCore.lambda$new$2(SolrCore.java:1048)
        at app//org.apache.solr.core.SolrCore$$Lambda$698/0x000000080127e018.call(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.FutureTask.run(FutureTask.java:264)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$234/0x0000000800ed54e8.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
	at __randomizedtesting.SeedInfo.seed([91F5FD1ED110E740]:0)


FAILED:  org.apache.solr.core.TestLazyCores.testDontEvictUsedCore

Error Message:
java.lang.AssertionError

Stack Trace:
java.lang.AssertionError
	at __randomizedtesting.SeedInfo.seed([91F5FD1ED110E740:E6663CC78AA5BFC4]:0)
	at org.junit.Assert.fail(Assert.java:87)
	at org.junit.Assert.assertTrue(Assert.java:42)
	at org.junit.Assert.assertTrue(Assert.java:53)
	at org.apache.solr.core.TestLazyCores.testDontEvictUsedCore(TestLazyCores.java:937)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:78)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:567)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:831)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
java.security.AccessControlException: access denied ("java.io.FilePermission" "/Users/jenkins/workspace/Solr-main-MacOSX/solr/core/build/resources/test/solr/filestore" "write")

Stack Trace:
java.security.AccessControlException: access denied ("java.io.FilePermission" "/Users/jenkins/workspace/Solr-main-MacOSX/solr/core/build/resources/test/solr/filestore" "write")
	at __randomizedtesting.SeedInfo.seed([91F5FD1ED110E740]:0)
	at java.base/java.security.AccessControlContext.checkPermission(AccessControlContext.java:472)
	at java.base/java.security.AccessController.checkPermission(AccessController.java:1036)
	at java.base/java.lang.SecurityManager.checkPermission(SecurityManager.java:408)
	at java.base/java.lang.SecurityManager.checkWrite(SecurityManager.java:838)
	at java.base/sun.nio.fs.UnixPath.checkWrite(UnixPath.java:816)
	at java.base/sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:388)
	at java.base/java.nio.file.Files.createDirectory(Files.java:694)
	at java.base/java.nio.file.Files.createAndCheckIsDirectory(Files.java:801)
	at java.base/java.nio.file.Files.createDirectories(Files.java:747)
	at org.apache.solr.filestore.DistribPackageStore.getPackageStoreDirPath(DistribPackageStore.java:584)
	at org.apache.solr.filestore.DistribPackageStore.ensurePackageStoreDir(DistribPackageStore.java:567)
	at org.apache.solr.filestore.DistribPackageStore.<init>(DistribPackageStore.java:83)
	at org.apache.solr.filestore.PackageStoreAPI.<init>(PackageStoreAPI.java:76)
	at org.apache.solr.core.CoreContainer.load(CoreContainer.java:725)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:184)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:156)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:162)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:112)
	at org.apache.solr.SolrTestCaseJ4.createCore(SolrTestCaseJ4.java:813)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:803)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:650)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:639)
	at org.apache.solr.cloud.AbstractZkTestCase.azt_beforeClass(AbstractZkTestCase.java:68)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:78)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:567)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:882)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:831)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 8 threads leaked from SUITE scope at org.apache.solr.rest.TestManagedResourceStorage: 
   1) Thread[id=19971, name=OverseerStateUpdate-72078747680833539-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:294)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   2) Thread[id=19961, name=httpShardExecutor-12889-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800ec3a58.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$260/0x0000000800f88000.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   3) Thread[id=19968, name=SUITE-TestManagedResourceStorage-seed#[91F5FD1ED110E740]-worker-SendThread(127.0.0.1:63111), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:214)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1380)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanAndNotifyState(ClientCnxn.java:1321)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1299)
   4) Thread[id=19969, name=SUITE-TestManagedResourceStorage-seed#[91F5FD1ED110E740]-worker-EventThread, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:551)
   5) Thread[id=19964, name=h2sc-12891-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800ec3a58.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$260/0x0000000800f88000.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   6) Thread[id=19972, name=OverseerCollectionConfigSetProcessor-72078747680833539-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:271)
        at app//org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:258)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   7) Thread[id=19970, name=zkConnectionManagerCallback-12896-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   8) Thread[id=19973, name=zkCallback-12895-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 8 threads leaked from SUITE scope at org.apache.solr.rest.TestManagedResourceStorage: 
   1) Thread[id=19971, name=OverseerStateUpdate-72078747680833539-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:294)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   2) Thread[id=19961, name=httpShardExecutor-12889-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800ec3a58.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$260/0x0000000800f88000.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   3) Thread[id=19968, name=SUITE-TestManagedResourceStorage-seed#[91F5FD1ED110E740]-worker-SendThread(127.0.0.1:63111), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:214)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1380)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanAndNotifyState(ClientCnxn.java:1321)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1299)
   4) Thread[id=19969, name=SUITE-TestManagedResourceStorage-seed#[91F5FD1ED110E740]-worker-EventThread, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:551)
   5) Thread[id=19964, name=h2sc-12891-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800ec3a58.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$260/0x0000000800f88000.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   6) Thread[id=19972, name=OverseerCollectionConfigSetProcessor-72078747680833539-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:271)
        at app//org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:258)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   7) Thread[id=19970, name=zkConnectionManagerCallback-12896-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   8) Thread[id=19973, name=zkCallback-12895-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
	at __randomizedtesting.SeedInfo.seed([91F5FD1ED110E740]:0)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=19961, name=httpShardExecutor-12889-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800ec3a58.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$260/0x0000000800f88000.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   2) Thread[id=19968, name=SUITE-TestManagedResourceStorage-seed#[91F5FD1ED110E740]-worker-SendThread(127.0.0.1:63111), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:362)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1197)
   3) Thread[id=19964, name=h2sc-12891-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800ec3a58.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$260/0x0000000800f88000.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   4) Thread[id=19970, name=zkConnectionManagerCallback-12896-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   5) Thread[id=19973, name=zkCallback-12895-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=19961, name=httpShardExecutor-12889-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800ec3a58.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$260/0x0000000800f88000.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   2) Thread[id=19968, name=SUITE-TestManagedResourceStorage-seed#[91F5FD1ED110E740]-worker-SendThread(127.0.0.1:63111), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:362)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1197)
   3) Thread[id=19964, name=h2sc-12891-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800ec3a58.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$260/0x0000000800f88000.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   4) Thread[id=19970, name=zkConnectionManagerCallback-12896-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   5) Thread[id=19973, name=zkCallback-12895-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
	at __randomizedtesting.SeedInfo.seed([91F5FD1ED110E740]:0)

[JENKINS] Solr-main-MacOSX (64bit/jdk-17.0.2) - Build # 658 - Failure!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/658/
Java: 64bit/jdk-17.0.2 -XX:-UseCompressedOops -XX:+UseParallelGC

7 tests failed.
FAILED:  org.apache.solr.cloud.TestRebalanceLeaders.testRebalanceLeaders

Error Message:
org.apache.solr.client.solrj.SolrServerException: Timeout occurred while waiting response from server at: https://127.0.0.1:55057/solr

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occurred while waiting response from server at: https://127.0.0.1:55057/solr
	at __randomizedtesting.SeedInfo.seed([6609FE0EF9AB700B:78E6CCFEC1003C7B]:0)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:679)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:239)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:222)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:389)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:359)
	at app//org.apache.solr.client.solrj.impl.BaseCloudSolrClient.sendRequest(BaseCloudSolrClient.java:1134)
	at app//org.apache.solr.client.solrj.impl.BaseCloudSolrClient.requestWithRetryOnStaleState(BaseCloudSolrClient.java:898)
	at app//org.apache.solr.client.solrj.impl.BaseCloudSolrClient.request(BaseCloudSolrClient.java:832)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:226)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:243)
	at app//org.apache.solr.cloud.TestRebalanceLeaders.rebalanceLeaderUsingSolrJAPI(TestRebalanceLeaders.java:373)
	at app//org.apache.solr.cloud.TestRebalanceLeaders.sendRebalanceCommand(TestRebalanceLeaders.java:279)
	at app//org.apache.solr.cloud.TestRebalanceLeaders.testRebalanceLeaders(TestRebalanceLeaders.java:135)
	at java.base@17.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base@17.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base@17.0.2/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base@17.0.2/java.lang.reflect.Method.invoke(Method.java:568)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.base/sun.nio.ch.NioSocketImpl.timedRead(NioSocketImpl.java:283)
	at java.base/sun.nio.ch.NioSocketImpl.implRead(NioSocketImpl.java:309)
	at java.base/sun.nio.ch.NioSocketImpl.read(NioSocketImpl.java:350)
	at java.base/sun.nio.ch.NioSocketImpl$1.read(NioSocketImpl.java:803)
	at java.base/java.net.Socket$SocketInputStream.read(Socket.java:966)
	at java.base/sun.security.ssl.SSLSocketInputRecord.read(SSLSocketInputRecord.java:478)
	at java.base/sun.security.ssl.SSLSocketInputRecord.readHeader(SSLSocketInputRecord.java:472)
	at java.base/sun.security.ssl.SSLSocketInputRecord.bytesInCompletePacket(SSLSocketInputRecord.java:70)
	at java.base/sun.security.ssl.SSLSocketImpl.readApplicationRecord(SSLSocketImpl.java:1455)
	at java.base/sun.security.ssl.SSLSocketImpl$AppInputStream.read(SSLSocketImpl.java:1059)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:280)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:157)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:551)
	... 58 more


FAILED:  org.apache.solr.core.TestLazyCores.testDontEvictUsedCore

Error Message:
java.lang.AssertionError

Stack Trace:
java.lang.AssertionError
	at __randomizedtesting.SeedInfo.seed([6609FE0EF9AB700B:119A3FD7A21E288F]:0)
	at org.junit.Assert.fail(Assert.java:87)
	at org.junit.Assert.assertTrue(Assert.java:42)
	at org.junit.Assert.assertTrue(Assert.java:53)
	at org.apache.solr.core.TestLazyCores.testDontEvictUsedCore(TestLazyCores.java:937)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:568)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:833)


FAILED:  org.apache.solr.handler.TestSolrConfigHandlerConcurrent.test

Error Message:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:49520/u: Underlying core creation failed while creating collection: control_collection

Stack Trace:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:49520/u: Underlying core creation failed while creating collection: control_collection
	at __randomizedtesting.SeedInfo.seed([6609FE0EF9AB700B:EE5DC1D457571DF3]:0)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:668)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:239)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:222)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:389)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:359)
	at app//org.apache.solr.client.solrj.impl.BaseCloudSolrClient.sendRequest(BaseCloudSolrClient.java:1134)
	at app//org.apache.solr.client.solrj.impl.BaseCloudSolrClient.requestWithRetryOnStaleState(BaseCloudSolrClient.java:898)
	at app//org.apache.solr.client.solrj.impl.BaseCloudSolrClient.request(BaseCloudSolrClient.java:832)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:226)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:243)
	at app//org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:339)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1079)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1052)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)


FAILED:  org.apache.solr.handler.component.DistributedQueryComponentCustomSortTest.test

Error Message:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:52243/collection1: org.apache.solr.client.solrj.SolrServerException: Connection refused

Stack Trace:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:52243/collection1: org.apache.solr.client.solrj.SolrServerException: Connection refused
	at __randomizedtesting.SeedInfo.seed([6609FE0EF9AB700B:EE5DC1D457571DF3]:0)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:668)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:239)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:222)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:226)
	at app//org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:1004)
	at app//org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:1019)
	at app//org.apache.solr.BaseDistributedSearchTestCase.queryServer(BaseDistributedSearchTestCase.java:623)
	at app//org.apache.solr.handler.component.DistributedQueryComponentCustomSortTest.queryAllowPartialResults(DistributedQueryComponentCustomSortTest.java:143)
	at app//org.apache.solr.handler.component.DistributedQueryComponentCustomSortTest.test(DistributedQueryComponentCustomSortTest.java:133)
	at java.base@17.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base@17.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base@17.0.2/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base@17.0.2/java.lang.reflect.Method.invoke(Method.java:568)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1081)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1052)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
java.security.AccessControlException: access denied ("java.io.FilePermission" "/Users/jenkins/workspace/Solr-main-MacOSX/solr/core/build/resources/test/solr/filestore" "write")

Stack Trace:
java.security.AccessControlException: access denied ("java.io.FilePermission" "/Users/jenkins/workspace/Solr-main-MacOSX/solr/core/build/resources/test/solr/filestore" "write")
	at __randomizedtesting.SeedInfo.seed([6609FE0EF9AB700B]:0)
	at java.base/java.security.AccessControlContext.checkPermission(AccessControlContext.java:485)
	at java.base/java.security.AccessController.checkPermission(AccessController.java:1068)
	at java.base/java.lang.SecurityManager.checkPermission(SecurityManager.java:416)
	at java.base/java.lang.SecurityManager.checkWrite(SecurityManager.java:847)
	at java.base/sun.nio.fs.UnixPath.checkWrite(UnixPath.java:787)
	at java.base/sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:389)
	at java.base/java.nio.file.Files.createDirectory(Files.java:700)
	at java.base/java.nio.file.Files.createAndCheckIsDirectory(Files.java:807)
	at java.base/java.nio.file.Files.createDirectories(Files.java:753)
	at org.apache.solr.filestore.DistribPackageStore.getPackageStoreDirPath(DistribPackageStore.java:584)
	at org.apache.solr.filestore.DistribPackageStore.ensurePackageStoreDir(DistribPackageStore.java:567)
	at org.apache.solr.filestore.DistribPackageStore.<init>(DistribPackageStore.java:83)
	at org.apache.solr.filestore.PackageStoreAPI.<init>(PackageStoreAPI.java:76)
	at org.apache.solr.core.CoreContainer.load(CoreContainer.java:725)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:184)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:156)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:162)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:112)
	at org.apache.solr.SolrTestCaseJ4.createCore(SolrTestCaseJ4.java:813)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:803)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:650)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:639)
	at org.apache.solr.cloud.AbstractZkTestCase.azt_beforeClass(AbstractZkTestCase.java:68)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:568)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:882)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:833)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 8 threads leaked from SUITE scope at org.apache.solr.rest.TestManagedResourceStorage: 
   1) Thread[id=19784, name=httpShardExecutor-12878-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@17.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$226/0x0000000800ee3a58.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$261/0x0000000800f906c8.run(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   2) Thread[id=19793, name=zkConnectionManagerCallback-12885-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3463)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3434)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at java.base@17.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   3) Thread[id=19798, name=zkCallback-12884-thread-3, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@17.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   4) Thread[id=19795, name=OverseerCollectionConfigSetProcessor-72074171899052035-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:271)
        at app//org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:258)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   5) Thread[id=19797, name=zkCallback-12884-thread-2, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@17.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   6) Thread[id=19792, name=SUITE-TestManagedResourceStorage-seed#[6609FE0EF9AB700B]-worker-EventThread, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3463)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3434)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at java.base@17.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:551)
   7) Thread[id=19791, name=SUITE-TestManagedResourceStorage-seed#[6609FE0EF9AB700B]-worker-SendThread(127.0.0.1:49420), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:362)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1197)
   8) Thread[id=19794, name=OverseerStateUpdate-72074171899052035-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:294)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   9) Thread[id=19787, name=h2sc-12880-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@17.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$226/0x0000000800ee3a58.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$261/0x0000000800f906c8.run(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
  10) Thread[id=19796, name=zkCallback-12884-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@17.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 8 threads leaked from SUITE scope at org.apache.solr.rest.TestManagedResourceStorage: 
   1) Thread[id=19784, name=httpShardExecutor-12878-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@17.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$226/0x0000000800ee3a58.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$261/0x0000000800f906c8.run(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   2) Thread[id=19793, name=zkConnectionManagerCallback-12885-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3463)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3434)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at java.base@17.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   3) Thread[id=19798, name=zkCallback-12884-thread-3, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@17.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   4) Thread[id=19795, name=OverseerCollectionConfigSetProcessor-72074171899052035-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:271)
        at app//org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:258)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   5) Thread[id=19797, name=zkCallback-12884-thread-2, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@17.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   6) Thread[id=19792, name=SUITE-TestManagedResourceStorage-seed#[6609FE0EF9AB700B]-worker-EventThread, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3463)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3434)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at java.base@17.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:551)
   7) Thread[id=19791, name=SUITE-TestManagedResourceStorage-seed#[6609FE0EF9AB700B]-worker-SendThread(127.0.0.1:49420), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:362)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1197)
   8) Thread[id=19794, name=OverseerStateUpdate-72074171899052035-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1672)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:294)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   9) Thread[id=19787, name=h2sc-12880-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@17.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$226/0x0000000800ee3a58.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$261/0x0000000800f906c8.run(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
  10) Thread[id=19796, name=zkCallback-12884-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@17.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
	at __randomizedtesting.SeedInfo.seed([6609FE0EF9AB700B]:0)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=19784, name=httpShardExecutor-12878-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@17.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$226/0x0000000800ee3a58.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$261/0x0000000800f906c8.run(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   2) Thread[id=19791, name=SUITE-TestManagedResourceStorage-seed#[6609FE0EF9AB700B]-worker-SendThread(127.0.0.1:49420), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1136)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1200)
   3) Thread[id=19793, name=zkConnectionManagerCallback-12885-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3463)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3434)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at java.base@17.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   4) Thread[id=19798, name=zkCallback-12884-thread-3, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@17.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   5) Thread[id=19787, name=h2sc-12880-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@17.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$226/0x0000000800ee3a58.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$261/0x0000000800f906c8.run(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   6) Thread[id=19796, name=zkCallback-12884-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@17.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   7) Thread[id=19797, name=zkCallback-12884-thread-2, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@17.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=19784, name=httpShardExecutor-12878-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@17.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$226/0x0000000800ee3a58.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$261/0x0000000800f906c8.run(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   2) Thread[id=19791, name=SUITE-TestManagedResourceStorage-seed#[6609FE0EF9AB700B]-worker-SendThread(127.0.0.1:49420), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1136)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1200)
   3) Thread[id=19793, name=zkConnectionManagerCallback-12885-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3463)
        at java.base@17.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3434)
        at java.base@17.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1623)
        at java.base@17.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   4) Thread[id=19798, name=zkCallback-12884-thread-3, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@17.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   5) Thread[id=19787, name=h2sc-12880-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@17.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@17.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$226/0x0000000800ee3a58.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$261/0x0000000800f906c8.run(Unknown Source)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   6) Thread[id=19796, name=zkCallback-12884-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@17.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
   7) Thread[id=19797, name=zkCallback-12884-thread-2, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@17.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@17.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@17.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401)
        at java.base@17.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
        at java.base@17.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@17.0.2/java.lang.Thread.run(Thread.java:833)
	at __randomizedtesting.SeedInfo.seed([6609FE0EF9AB700B]:0)

[JENKINS] Solr-main-MacOSX (64bit/jdk-11.0.13) - Build # 657 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/657/
Java: 64bit/jdk-11.0.13 -XX:+UseCompressedOops -XX:+UseSerialGC

4 tests failed.
FAILED:  org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=5592, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:156)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:89)
        at app//org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:306)
        at app//org.apache.solr.cloud.ZkController.pathExists(ZkController.java:1139)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:80)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getSnowballWordSet(AbstractAnalysisFactory.java:321)
        at app//org.apache.lucene.analysis.en.AbstractWordsFileFilterFactory.inform(AbstractWordsFileFilterFactory.java:90)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:723)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1341)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1194)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1179)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:615)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:177)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:107)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:244)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:314)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:241)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$587/0x0000000800694c40.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:46)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:239)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1686)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3134)
        at app//org.apache.solr.core.SolrCore$$Lambda$710/0x0000000800810440.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2499)
        at app//org.apache.solr.cloud.ZkController$$Lambda$828/0x0000000800897c40.run(Unknown Source)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=5592, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:156)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:89)
        at app//org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:306)
        at app//org.apache.solr.cloud.ZkController.pathExists(ZkController.java:1139)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:80)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getSnowballWordSet(AbstractAnalysisFactory.java:321)
        at app//org.apache.lucene.analysis.en.AbstractWordsFileFilterFactory.inform(AbstractWordsFileFilterFactory.java:90)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:723)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1341)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1194)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1179)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:615)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:177)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:107)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:244)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:314)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:241)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$587/0x0000000800694c40.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:46)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:239)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1686)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3134)
        at app//org.apache.solr.core.SolrCore$$Lambda$710/0x0000000800810440.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2499)
        at app//org.apache.solr.cloud.ZkController$$Lambda$828/0x0000000800897c40.run(Unknown Source)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
	at __randomizedtesting.SeedInfo.seed([CBF3205890E389BA]:0)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
java.security.AccessControlException: access denied ("java.io.FilePermission" "/Users/jenkins/workspace/Solr-main-MacOSX/solr/core/build/resources/test/solr/filestore" "write")

Stack Trace:
java.security.AccessControlException: access denied ("java.io.FilePermission" "/Users/jenkins/workspace/Solr-main-MacOSX/solr/core/build/resources/test/solr/filestore" "write")
	at __randomizedtesting.SeedInfo.seed([CBF3205890E389BA]:0)
	at java.base/java.security.AccessControlContext.checkPermission(AccessControlContext.java:472)
	at java.base/java.security.AccessController.checkPermission(AccessController.java:897)
	at java.base/java.lang.SecurityManager.checkPermission(SecurityManager.java:322)
	at java.base/java.lang.SecurityManager.checkWrite(SecurityManager.java:752)
	at java.base/sun.nio.fs.UnixPath.checkWrite(UnixPath.java:824)
	at java.base/sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:381)
	at java.base/java.nio.file.Files.createDirectory(Files.java:690)
	at java.base/java.nio.file.Files.createAndCheckIsDirectory(Files.java:797)
	at java.base/java.nio.file.Files.createDirectories(Files.java:743)
	at org.apache.solr.filestore.DistribPackageStore.getPackageStoreDirPath(DistribPackageStore.java:584)
	at org.apache.solr.filestore.DistribPackageStore.ensurePackageStoreDir(DistribPackageStore.java:567)
	at org.apache.solr.filestore.DistribPackageStore.<init>(DistribPackageStore.java:83)
	at org.apache.solr.filestore.PackageStoreAPI.<init>(PackageStoreAPI.java:76)
	at org.apache.solr.core.CoreContainer.load(CoreContainer.java:725)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:184)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:156)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:162)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:112)
	at org.apache.solr.SolrTestCaseJ4.createCore(SolrTestCaseJ4.java:813)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:803)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:650)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:639)
	at org.apache.solr.cloud.AbstractZkTestCase.azt_beforeClass(AbstractZkTestCase.java:68)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:882)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:829)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 8 threads leaked from SUITE scope at org.apache.solr.rest.TestManagedResourceStorage: 
   1) Thread[id=19236, name=zkCallback-12605-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   2) Thread[id=19233, name=zkConnectionManagerCallback-12606-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   3) Thread[id=19224, name=httpShardExecutor-12599-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$207/0x00000008003ce040.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$246/0x0000000800498040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   4) Thread[id=19231, name=SUITE-TestManagedResourceStorage-seed#[CBF3205890E389BA]-worker-SendThread(127.0.0.1:52984), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:214)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1380)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanAndNotifyState(ClientCnxn.java:1321)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1299)
   5) Thread[id=19232, name=SUITE-TestManagedResourceStorage-seed#[CBF3205890E389BA]-worker-EventThread, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:551)
   6) Thread[id=19234, name=OverseerStateUpdate-72073330156503043-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:294)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   7) Thread[id=19235, name=OverseerCollectionConfigSetProcessor-72073330156503043-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:271)
        at app//org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:258)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   8) Thread[id=19227, name=h2sc-12601-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$207/0x00000008003ce040.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$246/0x0000000800498040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 8 threads leaked from SUITE scope at org.apache.solr.rest.TestManagedResourceStorage: 
   1) Thread[id=19236, name=zkCallback-12605-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   2) Thread[id=19233, name=zkConnectionManagerCallback-12606-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   3) Thread[id=19224, name=httpShardExecutor-12599-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$207/0x00000008003ce040.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$246/0x0000000800498040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   4) Thread[id=19231, name=SUITE-TestManagedResourceStorage-seed#[CBF3205890E389BA]-worker-SendThread(127.0.0.1:52984), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:214)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1380)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanAndNotifyState(ClientCnxn.java:1321)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1299)
   5) Thread[id=19232, name=SUITE-TestManagedResourceStorage-seed#[CBF3205890E389BA]-worker-EventThread, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:551)
   6) Thread[id=19234, name=OverseerStateUpdate-72073330156503043-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:294)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   7) Thread[id=19235, name=OverseerCollectionConfigSetProcessor-72073330156503043-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:271)
        at app//org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:258)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   8) Thread[id=19227, name=h2sc-12601-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$207/0x00000008003ce040.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$246/0x0000000800498040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
	at __randomizedtesting.SeedInfo.seed([CBF3205890E389BA]:0)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=19236, name=zkCallback-12605-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   2) Thread[id=19233, name=zkConnectionManagerCallback-12606-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   3) Thread[id=19224, name=httpShardExecutor-12599-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$207/0x00000008003ce040.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$246/0x0000000800498040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   4) Thread[id=19231, name=SUITE-TestManagedResourceStorage-seed#[CBF3205890E389BA]-worker-SendThread(127.0.0.1:52984), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:362)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1197)
   5) Thread[id=19227, name=h2sc-12601-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$207/0x00000008003ce040.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$246/0x0000000800498040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=19236, name=zkCallback-12605-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   2) Thread[id=19233, name=zkConnectionManagerCallback-12606-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   3) Thread[id=19224, name=httpShardExecutor-12599-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$207/0x00000008003ce040.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$246/0x0000000800498040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   4) Thread[id=19231, name=SUITE-TestManagedResourceStorage-seed#[CBF3205890E389BA]-worker-SendThread(127.0.0.1:52984), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:362)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1197)
   5) Thread[id=19227, name=h2sc-12601-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$207/0x00000008003ce040.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$246/0x0000000800498040.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
	at __randomizedtesting.SeedInfo.seed([CBF3205890E389BA]:0)

[JENKINS] Solr-main-MacOSX (64bit/jdk-16.0.2) - Build # 656 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/656/
Java: 64bit/jdk-16.0.2 -XX:-UseCompressedOops -XX:+UseParallelGC

6 tests failed.
FAILED:  org.apache.solr.bench.MiniClusterBenchStateTest.testMiniClusterState

Error Message:
java.lang.Exception: Test abandoned because suite timeout was reached.

Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
	at __randomizedtesting.SeedInfo.seed([1D1D3BD436C53D66]:0)


FAILED:  org.apache.solr.bench.MiniClusterBenchStateTest.classMethod

Error Message:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).

Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
	at __randomizedtesting.SeedInfo.seed([1D1D3BD436C53D66]:0)


FAILED:  org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=4410, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:156)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:89)
        at app//org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:342)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:82)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getLines(AbstractAnalysisFactory.java:302)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getWordSet(AbstractAnalysisFactory.java:293)
        at app//org.apache.lucene.analysis.en.AbstractWordsFileFilterFactory.inform(AbstractWordsFileFilterFactory.java:88)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:723)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1341)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1194)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1179)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:615)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:177)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:107)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:244)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:314)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:241)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$602/0x0000000801130670.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:46)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:239)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1686)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3134)
        at app//org.apache.solr.core.SolrCore$$Lambda$725/0x0000000801295bf0.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2499)
        at app//org.apache.solr.cloud.ZkController$$Lambda$838/0x00000008012e7360.run(Unknown Source)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=4410, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:156)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:89)
        at app//org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:342)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:82)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getLines(AbstractAnalysisFactory.java:302)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getWordSet(AbstractAnalysisFactory.java:293)
        at app//org.apache.lucene.analysis.en.AbstractWordsFileFilterFactory.inform(AbstractWordsFileFilterFactory.java:88)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:723)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1341)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1194)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1179)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:615)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:177)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:107)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:244)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:314)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:241)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$602/0x0000000801130670.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:46)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:239)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1686)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3134)
        at app//org.apache.solr.core.SolrCore$$Lambda$725/0x0000000801295bf0.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2499)
        at app//org.apache.solr.cloud.ZkController$$Lambda$838/0x00000008012e7360.run(Unknown Source)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
	at __randomizedtesting.SeedInfo.seed([1D1D3BD436C53D66]:0)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
java.security.AccessControlException: access denied ("java.io.FilePermission" "/Users/jenkins/workspace/Solr-main-MacOSX/solr/core/build/resources/test/solr/filestore" "write")

Stack Trace:
java.security.AccessControlException: access denied ("java.io.FilePermission" "/Users/jenkins/workspace/Solr-main-MacOSX/solr/core/build/resources/test/solr/filestore" "write")
	at __randomizedtesting.SeedInfo.seed([1D1D3BD436C53D66]:0)
	at java.base/java.security.AccessControlContext.checkPermission(AccessControlContext.java:472)
	at java.base/java.security.AccessController.checkPermission(AccessController.java:1036)
	at java.base/java.lang.SecurityManager.checkPermission(SecurityManager.java:408)
	at java.base/java.lang.SecurityManager.checkWrite(SecurityManager.java:838)
	at java.base/sun.nio.fs.UnixPath.checkWrite(UnixPath.java:816)
	at java.base/sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:388)
	at java.base/java.nio.file.Files.createDirectory(Files.java:694)
	at java.base/java.nio.file.Files.createAndCheckIsDirectory(Files.java:801)
	at java.base/java.nio.file.Files.createDirectories(Files.java:747)
	at org.apache.solr.filestore.DistribPackageStore.getPackageStoreDirPath(DistribPackageStore.java:584)
	at org.apache.solr.filestore.DistribPackageStore.ensurePackageStoreDir(DistribPackageStore.java:567)
	at org.apache.solr.filestore.DistribPackageStore.<init>(DistribPackageStore.java:83)
	at org.apache.solr.filestore.PackageStoreAPI.<init>(PackageStoreAPI.java:76)
	at org.apache.solr.core.CoreContainer.load(CoreContainer.java:725)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:184)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:156)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:162)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:112)
	at org.apache.solr.SolrTestCaseJ4.createCore(SolrTestCaseJ4.java:813)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:803)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:650)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:639)
	at org.apache.solr.cloud.AbstractZkTestCase.azt_beforeClass(AbstractZkTestCase.java:68)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:78)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:567)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:882)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:831)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 8 threads leaked from SUITE scope at org.apache.solr.rest.TestManagedResourceStorage: 
   1) Thread[id=19639, name=OverseerStateUpdate-72072097965801475-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:294)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   2) Thread[id=19636, name=SUITE-TestManagedResourceStorage-seed#[1D1D3BD436C53D66]-worker-SendThread(127.0.0.1:65252), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:214)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1380)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanAndNotifyState(ClientCnxn.java:1321)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1299)
   3) Thread[id=19629, name=httpShardExecutor-12837-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edd2b0.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$260/0x0000000800f894c8.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   4) Thread[id=19632, name=h2sc-12839-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edd2b0.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$260/0x0000000800f894c8.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   5) Thread[id=19637, name=SUITE-TestManagedResourceStorage-seed#[1D1D3BD436C53D66]-worker-EventThread, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:551)
   6) Thread[id=19641, name=zkCallback-12843-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   7) Thread[id=19638, name=zkConnectionManagerCallback-12844-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   8) Thread[id=19640, name=OverseerCollectionConfigSetProcessor-72072097965801475-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:271)
        at app//org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:258)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 8 threads leaked from SUITE scope at org.apache.solr.rest.TestManagedResourceStorage: 
   1) Thread[id=19639, name=OverseerStateUpdate-72072097965801475-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:294)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   2) Thread[id=19636, name=SUITE-TestManagedResourceStorage-seed#[1D1D3BD436C53D66]-worker-SendThread(127.0.0.1:65252), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:214)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1380)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanAndNotifyState(ClientCnxn.java:1321)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1299)
   3) Thread[id=19629, name=httpShardExecutor-12837-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edd2b0.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$260/0x0000000800f894c8.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   4) Thread[id=19632, name=h2sc-12839-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edd2b0.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$260/0x0000000800f894c8.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   5) Thread[id=19637, name=SUITE-TestManagedResourceStorage-seed#[1D1D3BD436C53D66]-worker-EventThread, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:551)
   6) Thread[id=19641, name=zkCallback-12843-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   7) Thread[id=19638, name=zkConnectionManagerCallback-12844-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   8) Thread[id=19640, name=OverseerCollectionConfigSetProcessor-72072097965801475-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:271)
        at app//org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:258)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
	at __randomizedtesting.SeedInfo.seed([1D1D3BD436C53D66]:0)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=19636, name=SUITE-TestManagedResourceStorage-seed#[1D1D3BD436C53D66]-worker-SendThread(127.0.0.1:65252), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1136)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1200)
   2) Thread[id=19629, name=httpShardExecutor-12837-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edd2b0.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$260/0x0000000800f894c8.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   3) Thread[id=19632, name=h2sc-12839-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edd2b0.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$260/0x0000000800f894c8.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   4) Thread[id=19641, name=zkCallback-12843-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   5) Thread[id=19638, name=zkConnectionManagerCallback-12844-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=19636, name=SUITE-TestManagedResourceStorage-seed#[1D1D3BD436C53D66]-worker-SendThread(127.0.0.1:65252), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1136)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1200)
   2) Thread[id=19629, name=httpShardExecutor-12837-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edd2b0.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$260/0x0000000800f894c8.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   3) Thread[id=19632, name=h2sc-12839-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edd2b0.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$260/0x0000000800f894c8.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   4) Thread[id=19641, name=zkCallback-12843-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   5) Thread[id=19638, name=zkConnectionManagerCallback-12844-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
	at __randomizedtesting.SeedInfo.seed([1D1D3BD436C53D66]:0)

[JENKINS] Solr-main-MacOSX (64bit/jdk-16.0.2) - Build # 655 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/655/
Java: 64bit/jdk-16.0.2 -XX:+UseCompressedOops -XX:+UseShenandoahGC

7 tests failed.
FAILED:  org.apache.solr.cloud.HttpPartitionOnCommitTest.test

Error Message:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:63822/mo_i: Underlying core creation failed while creating collection: c8n_1x3_commits

Stack Trace:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:63822/mo_i: Underlying core creation failed while creating collection: c8n_1x3_commits
	at __randomizedtesting.SeedInfo.seed([D225FC7905E7F4C0:5A71C3A3AB1B9938]:0)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:668)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:239)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:222)
	at app//org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1290)
	at app//org.apache.solr.cloud.AbstractBasicDistributedZkTestBase.createCollection(AbstractBasicDistributedZkTestBase.java:803)
	at app//org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1747)
	at app//org.apache.solr.cloud.HttpPartitionOnCommitTest.oneShardTest(HttpPartitionOnCommitTest.java:125)
	at app//org.apache.solr.cloud.HttpPartitionOnCommitTest.test(HttpPartitionOnCommitTest.java:66)
	at java.base@16.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base@16.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:78)
	at java.base@16.0.2/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base@16.0.2/java.lang.reflect.Method.invoke(Method.java:567)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1081)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1052)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)


FAILED:  org.apache.solr.cloud.TestLeaderElectionZkExpiry.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.cloud.TestLeaderElectionZkExpiry: 
   1) Thread[id=25593, name=zkConnectionManagerCallback-15365-thread-1-EventThread, state=BLOCKED, group=TGRP-TestLeaderElectionZkExpiry]
        at app//org.apache.solr.common.cloud.ZkStateReader.createClusterStateWatchersAndUpdate(ZkStateReader.java:404)
        at app//org.apache.solr.cloud.ZkController$1.command(ZkController.java:336)
        at app//org.apache.solr.common.cloud.ConnectionManager$1.update(ConnectionManager.java:170)
        at app//org.apache.solr.common.cloud.DefaultConnectionStrategy.reconnect(DefaultConnectionStrategy.java:57)
        at app//org.apache.solr.common.cloud.ConnectionManager.process(ConnectionManager.java:152)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:580)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:555)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.cloud.TestLeaderElectionZkExpiry: 
   1) Thread[id=25593, name=zkConnectionManagerCallback-15365-thread-1-EventThread, state=BLOCKED, group=TGRP-TestLeaderElectionZkExpiry]
        at app//org.apache.solr.common.cloud.ZkStateReader.createClusterStateWatchersAndUpdate(ZkStateReader.java:404)
        at app//org.apache.solr.cloud.ZkController$1.command(ZkController.java:336)
        at app//org.apache.solr.common.cloud.ConnectionManager$1.update(ConnectionManager.java:170)
        at app//org.apache.solr.common.cloud.DefaultConnectionStrategy.reconnect(DefaultConnectionStrategy.java:57)
        at app//org.apache.solr.common.cloud.ConnectionManager.process(ConnectionManager.java:152)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:580)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:555)
	at __randomizedtesting.SeedInfo.seed([D225FC7905E7F4C0]:0)


FAILED:  org.apache.solr.core.TestLazyCores.testDontEvictUsedCore

Error Message:
java.lang.AssertionError

Stack Trace:
java.lang.AssertionError
	at __randomizedtesting.SeedInfo.seed([D225FC7905E7F4C0:A5B63DA05E52AC44]:0)
	at org.junit.Assert.fail(Assert.java:87)
	at org.junit.Assert.assertTrue(Assert.java:42)
	at org.junit.Assert.assertTrue(Assert.java:53)
	at org.apache.solr.core.TestLazyCores.testDontEvictUsedCore(TestLazyCores.java:937)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:78)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:567)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:831)


FAILED:  org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=4947, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:156)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:89)
        at app//org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:306)
        at app//org.apache.solr.cloud.ZkController.pathExists(ZkController.java:1139)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:80)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getSnowballWordSet(AbstractAnalysisFactory.java:321)
        at app//org.apache.lucene.analysis.en.AbstractWordsFileFilterFactory.inform(AbstractWordsFileFilterFactory.java:90)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:723)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1341)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1194)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1179)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:615)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:177)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:107)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:244)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:314)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:241)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$601/0x00000008011374f0.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:46)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:239)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1686)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3134)
        at app//org.apache.solr.core.SolrCore$$Lambda$724/0x000000080129c308.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2499)
        at app//org.apache.solr.cloud.ZkController$$Lambda$862/0x00000008012fc318.run(Unknown Source)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.designer.TestSchemaDesignerSettingsDAO: 
   1) Thread[id=4947, name=ZKEventListenerThread, state=TIMED_WAITING, group=TGRP-TestSchemaDesignerSettingsDAO]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:156)
        at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:89)
        at app//org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:306)
        at app//org.apache.solr.cloud.ZkController.pathExists(ZkController.java:1139)
        at app//org.apache.solr.cloud.ZkSolrResourceLoader.openResource(ZkSolrResourceLoader.java:80)
        at app//org.apache.lucene.analysis.AbstractAnalysisFactory.getSnowballWordSet(AbstractAnalysisFactory.java:321)
        at app//org.apache.lucene.analysis.en.AbstractWordsFileFilterFactory.inform(AbstractWordsFileFilterFactory.java:90)
        at app//org.apache.solr.core.SolrResourceLoader.informAware(SolrResourceLoader.java:723)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsInChain(ManagedIndexSchema.java:1341)
        at app//org.apache.solr.schema.ManagedIndexSchema.informResourceLoaderAwareObjectsForFieldType(ManagedIndexSchema.java:1194)
        at app//org.apache.solr.schema.ManagedIndexSchema.postReadInform(ManagedIndexSchema.java:1179)
        at app//org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:615)
        at app//org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:177)
        at app//org.apache.solr.schema.ManagedIndexSchema.<init>(ManagedIndexSchema.java:107)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:244)
        at app//org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:51)
        at app//org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:314)
        at app//org.apache.solr.core.ConfigSetService.lambda$loadConfigSet$0(ConfigSetService.java:241)
        at app//org.apache.solr.core.ConfigSetService$$Lambda$601/0x00000008011374f0.get(Unknown Source)
        at app//org.apache.solr.core.ConfigSet.<init>(ConfigSet.java:46)
        at app//org.apache.solr.core.ConfigSetService.loadConfigSet(ConfigSetService.java:239)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1686)
        at app//org.apache.solr.core.SolrCore.lambda$getConfListener$21(SolrCore.java:3134)
        at app//org.apache.solr.core.SolrCore$$Lambda$724/0x000000080129c308.run(Unknown Source)
        at app//org.apache.solr.cloud.ZkController.lambda$fireEventListeners$18(ZkController.java:2499)
        at app//org.apache.solr.cloud.ZkController$$Lambda$862/0x00000008012fc318.run(Unknown Source)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
	at __randomizedtesting.SeedInfo.seed([D225FC7905E7F4C0]:0)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
java.security.AccessControlException: access denied ("java.io.FilePermission" "/Users/jenkins/workspace/Solr-main-MacOSX/solr/core/build/resources/test/solr/filestore" "write")

Stack Trace:
java.security.AccessControlException: access denied ("java.io.FilePermission" "/Users/jenkins/workspace/Solr-main-MacOSX/solr/core/build/resources/test/solr/filestore" "write")
	at __randomizedtesting.SeedInfo.seed([D225FC7905E7F4C0]:0)
	at java.base/java.security.AccessControlContext.checkPermission(AccessControlContext.java:472)
	at java.base/java.security.AccessController.checkPermission(AccessController.java:1036)
	at java.base/java.lang.SecurityManager.checkPermission(SecurityManager.java:408)
	at java.base/java.lang.SecurityManager.checkWrite(SecurityManager.java:838)
	at java.base/sun.nio.fs.UnixPath.checkWrite(UnixPath.java:816)
	at java.base/sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:388)
	at java.base/java.nio.file.Files.createDirectory(Files.java:694)
	at java.base/java.nio.file.Files.createAndCheckIsDirectory(Files.java:801)
	at java.base/java.nio.file.Files.createDirectories(Files.java:747)
	at org.apache.solr.filestore.DistribPackageStore.getPackageStoreDirPath(DistribPackageStore.java:584)
	at org.apache.solr.filestore.DistribPackageStore.ensurePackageStoreDir(DistribPackageStore.java:567)
	at org.apache.solr.filestore.DistribPackageStore.<init>(DistribPackageStore.java:83)
	at org.apache.solr.filestore.PackageStoreAPI.<init>(PackageStoreAPI.java:76)
	at org.apache.solr.core.CoreContainer.load(CoreContainer.java:725)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:184)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:156)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:162)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:112)
	at org.apache.solr.SolrTestCaseJ4.createCore(SolrTestCaseJ4.java:813)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:803)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:650)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:639)
	at org.apache.solr.cloud.AbstractZkTestCase.azt_beforeClass(AbstractZkTestCase.java:68)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:78)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:567)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:882)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:831)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 8 threads leaked from SUITE scope at org.apache.solr.rest.TestManagedResourceStorage: 
   1) Thread[id=20023, name=h2sc-12919-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edb820.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$260/0x0000000800f88258.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   2) Thread[id=20029, name=zkConnectionManagerCallback-12924-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   3) Thread[id=20031, name=OverseerCollectionConfigSetProcessor-72071315558105091-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:271)
        at app//org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:258)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   4) Thread[id=20020, name=httpShardExecutor-12917-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edb820.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$260/0x0000000800f88258.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   5) Thread[id=20028, name=SUITE-TestManagedResourceStorage-seed#[D225FC7905E7F4C0]-worker-EventThread, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:551)
   6) Thread[id=20027, name=SUITE-TestManagedResourceStorage-seed#[D225FC7905E7F4C0]-worker-SendThread(127.0.0.1:56078), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:214)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1380)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanAndNotifyState(ClientCnxn.java:1321)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1299)
   7) Thread[id=20032, name=zkCallback-12923-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   8) Thread[id=20030, name=OverseerStateUpdate-72071315558105091-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:294)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 8 threads leaked from SUITE scope at org.apache.solr.rest.TestManagedResourceStorage: 
   1) Thread[id=20023, name=h2sc-12919-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edb820.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$260/0x0000000800f88258.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   2) Thread[id=20029, name=zkConnectionManagerCallback-12924-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   3) Thread[id=20031, name=OverseerCollectionConfigSetProcessor-72071315558105091-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:271)
        at app//org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:258)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   4) Thread[id=20020, name=httpShardExecutor-12917-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edb820.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$260/0x0000000800f88258.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   5) Thread[id=20028, name=SUITE-TestManagedResourceStorage-seed#[D225FC7905E7F4C0]-worker-EventThread, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:551)
   6) Thread[id=20027, name=SUITE-TestManagedResourceStorage-seed#[D225FC7905E7F4C0]-worker-SendThread(127.0.0.1:56078), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:214)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1380)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanAndNotifyState(ClientCnxn.java:1321)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1299)
   7) Thread[id=20032, name=zkCallback-12923-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   8) Thread[id=20030, name=OverseerStateUpdate-72071315558105091-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:294)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
	at __randomizedtesting.SeedInfo.seed([D225FC7905E7F4C0]:0)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=20023, name=h2sc-12919-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edb820.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$260/0x0000000800f88258.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   2) Thread[id=20029, name=zkConnectionManagerCallback-12924-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   3) Thread[id=20020, name=httpShardExecutor-12917-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edb820.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$260/0x0000000800f88258.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   4) Thread[id=20027, name=SUITE-TestManagedResourceStorage-seed#[D225FC7905E7F4C0]-worker-SendThread(127.0.0.1:56078), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1136)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1200)
   5) Thread[id=20032, name=zkCallback-12923-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=20023, name=h2sc-12919-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edb820.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$260/0x0000000800f88258.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   2) Thread[id=20029, name=zkConnectionManagerCallback-12924-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
        at java.base@16.0.2/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
        at java.base@16.0.2/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
        at java.base@16.0.2/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   3) Thread[id=20020, name=httpShardExecutor-12917-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@16.0.2/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
        at java.base@16.0.2/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$225/0x0000000800edb820.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$260/0x0000000800f88258.run(Unknown Source)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   4) Thread[id=20027, name=SUITE-TestManagedResourceStorage-seed#[D225FC7905E7F4C0]-worker-SendThread(127.0.0.1:56078), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1136)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1200)
   5) Thread[id=20032, name=zkCallback-12923-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@16.0.2/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@16.0.2/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
        at java.base@16.0.2/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
	at __randomizedtesting.SeedInfo.seed([D225FC7905E7F4C0]:0)

[JENKINS] Solr-main-MacOSX (64bit/jdk-11.0.13) - Build # 654 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/654/
Java: 64bit/jdk-11.0.13 -XX:+UseCompressedOops -XX:+UseConcMarkSweepGC

3 tests failed.
FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
java.security.AccessControlException: access denied ("java.io.FilePermission" "/Users/jenkins/workspace/Solr-main-MacOSX/solr/core/build/resources/test/solr/filestore" "write")

Stack Trace:
java.security.AccessControlException: access denied ("java.io.FilePermission" "/Users/jenkins/workspace/Solr-main-MacOSX/solr/core/build/resources/test/solr/filestore" "write")
	at __randomizedtesting.SeedInfo.seed([19A92430FFFE9B73]:0)
	at java.base/java.security.AccessControlContext.checkPermission(AccessControlContext.java:472)
	at java.base/java.security.AccessController.checkPermission(AccessController.java:897)
	at java.base/java.lang.SecurityManager.checkPermission(SecurityManager.java:322)
	at java.base/java.lang.SecurityManager.checkWrite(SecurityManager.java:752)
	at java.base/sun.nio.fs.UnixPath.checkWrite(UnixPath.java:824)
	at java.base/sun.nio.fs.UnixFileSystemProvider.createDirectory(UnixFileSystemProvider.java:381)
	at java.base/java.nio.file.Files.createDirectory(Files.java:690)
	at java.base/java.nio.file.Files.createAndCheckIsDirectory(Files.java:797)
	at java.base/java.nio.file.Files.createDirectories(Files.java:743)
	at org.apache.solr.filestore.DistribPackageStore.getPackageStoreDirPath(DistribPackageStore.java:584)
	at org.apache.solr.filestore.DistribPackageStore.ensurePackageStoreDir(DistribPackageStore.java:567)
	at org.apache.solr.filestore.DistribPackageStore.<init>(DistribPackageStore.java:83)
	at org.apache.solr.filestore.PackageStoreAPI.<init>(PackageStoreAPI.java:76)
	at org.apache.solr.core.CoreContainer.load(CoreContainer.java:725)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:184)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:156)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:162)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:112)
	at org.apache.solr.SolrTestCaseJ4.createCore(SolrTestCaseJ4.java:813)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:803)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:650)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:639)
	at org.apache.solr.cloud.AbstractZkTestCase.azt_beforeClass(AbstractZkTestCase.java:68)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:882)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:829)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 11 threads leaked from SUITE scope at org.apache.solr.rest.TestManagedResourceStorage: 
   1) Thread[id=22574, name=OverseerCollectionConfigSetProcessor-72070389680373763-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:271)
        at app//org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:258)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   2) Thread[id=22575, name=zkCallback-15736-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   3) Thread[id=22571, name=SUITE-TestManagedResourceStorage-seed#[19A92430FFFE9B73]-worker-EventThread, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:551)
   4) Thread[id=22572, name=zkConnectionManagerCallback-15737-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   5) Thread[id=22576, name=zkCallback-15736-thread-2, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   6) Thread[id=22566, name=h2sc-15732-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$207/0x00000008003cec40.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$247/0x0000000800498840.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   7) Thread[id=22577, name=zkCallback-15736-thread-3, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   8) Thread[id=22573, name=OverseerStateUpdate-72070389680373763-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:294)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   9) Thread[id=22578, name=zkCallback-15736-thread-4, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  10) Thread[id=22563, name=httpShardExecutor-15730-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$207/0x00000008003cec40.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$247/0x0000000800498840.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  11) Thread[id=22570, name=SUITE-TestManagedResourceStorage-seed#[19A92430FFFE9B73]-worker-SendThread(127.0.0.1:52968), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:362)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1197)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 11 threads leaked from SUITE scope at org.apache.solr.rest.TestManagedResourceStorage: 
   1) Thread[id=22574, name=OverseerCollectionConfigSetProcessor-72070389680373763-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:271)
        at app//org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:258)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   2) Thread[id=22575, name=zkCallback-15736-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   3) Thread[id=22571, name=SUITE-TestManagedResourceStorage-seed#[19A92430FFFE9B73]-worker-EventThread, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:551)
   4) Thread[id=22572, name=zkConnectionManagerCallback-15737-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   5) Thread[id=22576, name=zkCallback-15736-thread-2, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   6) Thread[id=22566, name=h2sc-15732-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$207/0x00000008003cec40.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$247/0x0000000800498840.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   7) Thread[id=22577, name=zkCallback-15736-thread-3, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   8) Thread[id=22573, name=OverseerStateUpdate-72070389680373763-127.0.0.1:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2123)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:477)
        at app//org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:294)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   9) Thread[id=22578, name=zkCallback-15736-thread-4, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  10) Thread[id=22563, name=httpShardExecutor-15730-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$207/0x00000008003cec40.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$247/0x0000000800498840.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
  11) Thread[id=22570, name=SUITE-TestManagedResourceStorage-seed#[19A92430FFFE9B73]-worker-SendThread(127.0.0.1:52968), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:362)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1197)
	at __randomizedtesting.SeedInfo.seed([19A92430FFFE9B73]:0)


FAILED:  org.apache.solr.rest.TestManagedResourceStorage.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=22566, name=h2sc-15732-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$207/0x00000008003cec40.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$247/0x0000000800498840.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   2) Thread[id=22575, name=zkCallback-15736-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   3) Thread[id=22577, name=zkCallback-15736-thread-3, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   4) Thread[id=22578, name=zkCallback-15736-thread-4, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   5) Thread[id=22572, name=zkConnectionManagerCallback-15737-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   6) Thread[id=22563, name=httpShardExecutor-15730-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$207/0x00000008003cec40.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$247/0x0000000800498840.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   7) Thread[id=22570, name=SUITE-TestManagedResourceStorage-seed#[19A92430FFFE9B73]-worker-SendThread(127.0.0.1:52968), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1136)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1200)
   8) Thread[id=22576, name=zkCallback-15736-thread-2, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=22566, name=h2sc-15732-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$207/0x00000008003cec40.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$247/0x0000000800498840.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   2) Thread[id=22575, name=zkCallback-15736-thread-1, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   3) Thread[id=22577, name=zkCallback-15736-thread-3, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   4) Thread[id=22578, name=zkCallback-15736-thread-4, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   5) Thread[id=22572, name=zkConnectionManagerCallback-15737-thread-1, state=WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.13/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.13/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   6) Thread[id=22563, name=httpShardExecutor-15730-thread-1, state=RUNNABLE, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/sun.nio.ch.KQueue.poll(Native Method)
        at java.base@11.0.13/sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:122)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:124)
        at java.base@11.0.13/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141)
        at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
        at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
        at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
        at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$207/0x00000008003cec40.run(Unknown Source)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:235)
        at app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$247/0x0000000800498840.run(Unknown Source)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
   7) Thread[id=22570, name=SUITE-TestManagedResourceStorage-seed#[19A92430FFFE9B73]-worker-SendThread(127.0.0.1:52968), state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1136)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1200)
   8) Thread[id=22576, name=zkCallback-15736-thread-2, state=TIMED_WAITING, group=TGRP-TestManagedResourceStorage]
        at java.base@11.0.13/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.13/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@11.0.13/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1053)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.13/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.13/java.lang.Thread.run(Thread.java:829)
	at __randomizedtesting.SeedInfo.seed([19A92430FFFE9B73]:0)