You are viewing a plain text version of this content. The canonical link for it is here.
Posted to builds@solr.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2023/09/03 10:07:33 UTC

[JENKINS] Solr-main-MacOSX (64bit/hotspot/jdk-19) - Build # 3272 - Still Unstable!

Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/3272/
Java: 64bit/hotspot/jdk-19 -XX:+UseCompressedOops -XX:+UseSerialGC

3 tests failed.
FAILED:  org.apache.solr.cluster.placement.impl.PlacementPluginIntegrationTest.testDynamicReconfiguration

Error Message:
java.lang.AssertionError

Stack Trace:
java.lang.AssertionError
	at __randomizedtesting.SeedInfo.seed([7F48EF0D74C0E2B1:852DA8AA3A6D708A]:0)
	at org.junit.Assert.fail(Assert.java:87)
	at org.junit.Assert.assertTrue(Assert.java:42)
	at org.junit.Assert.assertNotNull(Assert.java:713)
	at org.junit.Assert.assertNotNull(Assert.java:723)
	at org.apache.solr.cluster.placement.impl.PlacementPluginIntegrationTest.testDynamicReconfiguration(PlacementPluginIntegrationTest.java:169)
	at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
	at java.base/java.lang.reflect.Method.invoke(Method.java:578)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
	at java.base/java.lang.Thread.run(Thread.java:1589)


FAILED:  org.apache.solr.security.hadoop.SaslZkACLProviderTest.testSaslZkACLProvider

Error Message:
org.apache.solr.common.SolrException: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper localhost:52611/solr within 15000 ms

Stack Trace:
org.apache.solr.common.SolrException: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper localhost:52611/solr within 15000 ms
	at __randomizedtesting.SeedInfo.seed([7F48EF0D74C0E2B1:7BBCB9D5ED8441BF]:0)
	at app//org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:209)
	at app//org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:111)
	at app//org.apache.solr.security.hadoop.SaslZkACLProviderTest$SolrZkClientWithACLs.<init>(SaslZkACLProviderTest.java:182)
	at app//org.apache.solr.security.hadoop.SaslZkACLProviderTest.setupZNodes(SaslZkACLProviderTest.java:111)
	at app//org.apache.solr.security.hadoop.SaslZkACLProviderTest.setUp(SaslZkACLProviderTest.java:102)
	at java.base@19/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
	at java.base@19/java.lang.reflect.Method.invoke(Method.java:578)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:980)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
	at java.base@19/java.lang.Thread.run(Thread.java:1589)
Caused by: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper localhost:52611/solr within 15000 ms
	at org.apache.solr.common.cloud.ConnectionManager.waitForConnected(ConnectionManager.java:296)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:200)
	... 47 more


FAILED:  org.apache.solr.hdfs.cloud.api.collections.HdfsCloudIncrementalBackupTest.testRestoreToOriginalCollection

Error Message:
java.lang.AssertionError: expected:<COMPLETED> but was:<RUNNING>

Stack Trace:
java.lang.AssertionError: expected:<COMPLETED> but was:<RUNNING>
	at __randomizedtesting.SeedInfo.seed([7F48EF0D74C0E2B1:4ABC04569F1CD4EF]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.junit.Assert.failNotEquals(Assert.java:835)
	at org.junit.Assert.assertEquals(Assert.java:120)
	at org.junit.Assert.assertEquals(Assert.java:146)
	at org.apache.solr.cloud.api.collections.AbstractIncrementalBackupTest.testRestoreToOriginalCollection(AbstractIncrementalBackupTest.java:209)
	at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
	at java.base/java.lang.reflect.Method.invoke(Method.java:578)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
	at java.base/java.lang.Thread.run(Thread.java:1589)

[JENKINS] Solr-main-MacOSX (64bit/hotspot/jdk-17.0.5) - Build # 3277 - Still unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/3277/
Java: 64bit/hotspot/jdk-17.0.5 -XX:-UseCompressedOops -XX:+UseG1GC

2 tests failed.
FAILED:  org.apache.solr.metrics.reporters.SolrJmxReporterTest.testClosedCore

Error Message:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=12298, name=TestMBeanThread, state=RUNNABLE, group=TGRP-SolrJmxReporterTest]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=12298, name=TestMBeanThread, state=RUNNABLE, group=TGRP-SolrJmxReporterTest]
	at __randomizedtesting.SeedInfo.seed([A0686A797A9606F6:D715BC595A0F6406]:0)
Caused by: java.lang.AssertionError
	at __randomizedtesting.SeedInfo.seed([A0686A797A9606F6]:0)
	at org.junit.Assert.fail(Assert.java:87)
	at org.junit.Assert.assertTrue(Assert.java:42)
	at org.junit.Assert.assertNotNull(Assert.java:713)
	at org.junit.Assert.assertNotNull(Assert.java:723)
	at org.apache.solr.metrics.reporters.SolrJmxReporterTest.lambda$testClosedCore$3(SolrJmxReporterTest.java:239)
	at java.base/java.lang.Thread.run(Thread.java:833)


FAILED:  org.apache.solr.search.TestCoordinatorRole.testNRTRestart

Error Message:
java.lang.AssertionError: initial error time threshold exceeded

Stack Trace:
java.lang.AssertionError: initial error time threshold exceeded
	at __randomizedtesting.SeedInfo.seed([A0686A797A9606F6:50FCDAB2D88BD420]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.solr.search.TestCoordinatorRole.getHostCoreName(TestCoordinatorRole.java:477)
	at org.apache.solr.search.TestCoordinatorRole.testNRTRestart(TestCoordinatorRole.java:402)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:568)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
	at java.base/java.lang.Thread.run(Thread.java:833)

[JENKINS] Solr-main-MacOSX (64bit/hotspot/jdk-20) - Build # 3276 - Failure!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/3276/
Java: 64bit/hotspot/jdk-20 -XX:+UseCompressedOops -XX:+UseG1GC

No tests ran.

[JENKINS] Solr-main-MacOSX (64bit/hotspot/jdk-11.0.15) - Build # 3275 - Still unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/3275/
Java: 64bit/hotspot/jdk-11.0.15 -XX:+UseCompressedOops -XX:+UseConcMarkSweepGC

1 tests failed.
FAILED:  org.apache.solr.s3.S3IncrementalBackupTest.testRestoreToOriginalCollection

Error Message:
java.lang.AssertionError: expected:<COMPLETED> but was:<RUNNING>

Stack Trace:
java.lang.AssertionError: expected:<COMPLETED> but was:<RUNNING>
	at __randomizedtesting.SeedInfo.seed([34A84C93603128C9:15CA7C88BED1E97]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.junit.Assert.failNotEquals(Assert.java:835)
	at org.junit.Assert.assertEquals(Assert.java:120)
	at org.junit.Assert.assertEquals(Assert.java:146)
	at org.apache.solr.cloud.api.collections.AbstractIncrementalBackupTest.testRestoreToOriginalCollection(AbstractIncrementalBackupTest.java:224)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.adobe.testing.s3mock.junit4.S3MockRule$1.evaluate(S3MockRule.java:66)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
	at java.base/java.lang.Thread.run(Thread.java:829)

[JENKINS] Solr-main-MacOSX (64bit/hotspot/jdk-20) - Build # 3274 - Failure!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/3274/
Java: 64bit/hotspot/jdk-20 -XX:+UseCompressedOops -XX:+UseParallelGC

No tests ran.

[JENKINS] Solr-main-MacOSX (64bit/hotspot/jdk-11.0.15) - Build # 3273 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/3273/
Java: 64bit/hotspot/jdk-11.0.15 -XX:-UseCompressedOops -XX:+UseParallelGC

3 tests failed.
FAILED:  org.apache.solr.gcs.GCSIncrementalBackupTest.testRestoreToOriginalCollection

Error Message:
java.lang.AssertionError: expected:<COMPLETED> but was:<RUNNING>

Stack Trace:
java.lang.AssertionError: expected:<COMPLETED> but was:<RUNNING>
	at __randomizedtesting.SeedInfo.seed([80F9D56CC1404BF:3DFB760D27C832E1]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.junit.Assert.failNotEquals(Assert.java:835)
	at org.junit.Assert.assertEquals(Assert.java:120)
	at org.junit.Assert.assertEquals(Assert.java:146)
	at org.apache.solr.cloud.api.collections.AbstractIncrementalBackupTest.testRestoreToOriginalCollection(AbstractIncrementalBackupTest.java:224)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
	at java.base/java.lang.Thread.run(Thread.java:829)


FAILED:  org.apache.solr.gcs.GCSIncrementalBackupTest.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 6 threads leaked from SUITE scope at org.apache.solr.gcs.GCSIncrementalBackupTest: 
   1) Thread[id=57, name=qtp1436107811-57, state=RUNNABLE, group=TGRP-GCSIncrementalBackupTest]
        at java.base@11.0.15/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.15/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.15/java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:885)
        at java.base@11.0.15/java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireShared(AbstractQueuedSynchronizer.java:1009)
        at java.base@11.0.15/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireShared(AbstractQueuedSynchronizer.java:1324)
        at java.base@11.0.15/java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.lock(ReentrantReadWriteLock.java:738)
        at java.management@11.0.15/com.sun.jmx.mbeanserver.Repository.query(Repository.java:525)
        at java.management@11.0.15/com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.queryMBeansImpl(DefaultMBeanServerInterceptor.java:507)
        at java.management@11.0.15/com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.queryMBeans(DefaultMBeanServerInterceptor.java:476)
        at java.management@11.0.15/com.sun.jmx.mbeanserver.JmxMBeanServer.queryMBeans(JmxMBeanServer.java:592)
        at app//org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter$JmxListener.unregisterMBean(JmxMetricsReporter.java:567)
        at app//org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter$JmxListener.onGaugeRemoved(JmxMetricsReporter.java:602)
        at app//com.codahale.metrics.MetricRegistry.notifyListenerOfRemovedMetric(MetricRegistry.java:574)
        at app//com.codahale.metrics.MetricRegistry.onMetricRemoved(MetricRegistry.java:568)
        at app//com.codahale.metrics.MetricRegistry.remove(MetricRegistry.java:362)
        at app//com.codahale.metrics.MetricRegistry.removeMatching(MetricRegistry.java:376)
        at app//org.apache.solr.metrics.SolrMetricManager.unregisterGauges(SolrMetricManager.java:836)
        at app//org.apache.solr.metrics.SolrCoreMetricManager.close(SolrCoreMetricManager.java:160)
        at app//org.apache.solr.core.SolrCore.doClose(SolrCore.java:1769)
        at app//org.apache.solr.core.SolrCore.close(SolrCore.java:1755)
        at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1223)
        at app//org.apache.solr.core.SolrCore.reload(SolrCore.java:789)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:2009)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1977)
        at app//org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$2(CoreAdminOperation.java:136)
        at app//org.apache.solr.handler.admin.CoreAdminOperation$$Lambda$512/0x0000000147dc4440.execute(Unknown Source)
        at app//org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:406)
        at app//org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:365)
        at app//org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:228)
        at app//org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:224)
        at app//org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:922)
        at app//org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:870)
        at app//org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:539)
        at app//org.apache.solr.servlet.SolrDispatchFilter.dispatch(SolrDispatchFilter.java:248)
        at app//org.apache.solr.servlet.SolrDispatchFilter.lambda$doFilter$0(SolrDispatchFilter.java:215)
        at app//org.apache.solr.servlet.SolrDispatchFilter$$Lambda$751/0x0000000148585d60.run(Unknown Source)
        at app//org.apache.solr.servlet.ServletUtils.traceHttpRequestExecution2(ServletUtils.java:241)
        at app//org.apache.solr.servlet.ServletUtils.rateLimitRequest(ServletUtils.java:211)
        at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:209)
        at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:192)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.apache.solr.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:187)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1384)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1306)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at app//org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:301)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at app//org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:790)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at app//org.eclipse.jetty.server.Server.handle(Server.java:563)
        at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at app//org.eclipse.jetty.server.HttpChannel$$Lambda$749/0x0000000148ab3040.dispatch(Unknown Source)
        at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at app//org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:457)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:969)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.doRunJob(QueuedThreadPool.java:1194)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1149)
        at java.base@11.0.15/java.lang.Thread.run(Thread.java:829)
   2) Thread[id=243, name=searcherExecutor-173-thread-1, state=WAITING, group=TGRP-GCSIncrementalBackupTest]
        at java.base@11.0.15/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.15/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.15/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.15/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.15/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.15/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.15/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.15/java.lang.Thread.run(Thread.java:829)
   3) Thread[id=49, name=qtp1490440899-49, state=RUNNABLE, group=TGRP-GCSIncrementalBackupTest]
        at java.base@11.0.15/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.15/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.15/java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:885)
        at java.base@11.0.15/java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireShared(AbstractQueuedSynchronizer.java:1009)
        at java.base@11.0.15/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireShared(AbstractQueuedSynchronizer.java:1324)
        at java.base@11.0.15/java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.lock(ReentrantReadWriteLock.java:738)
        at java.management@11.0.15/com.sun.jmx.mbeanserver.Repository.query(Repository.java:525)
        at java.management@11.0.15/com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.queryMBeansImpl(DefaultMBeanServerInterceptor.java:507)
        at java.management@11.0.15/com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.queryMBeans(DefaultMBeanServerInterceptor.java:476)
        at java.management@11.0.15/com.sun.jmx.mbeanserver.JmxMBeanServer.queryMBeans(JmxMBeanServer.java:592)
        at app//org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter$JmxListener.unregisterMBean(JmxMetricsReporter.java:567)
        at app//org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter$JmxListener.onGaugeRemoved(JmxMetricsReporter.java:602)
        at app//com.codahale.metrics.MetricRegistry.notifyListenerOfRemovedMetric(MetricRegistry.java:574)
        at app//com.codahale.metrics.MetricRegistry.onMetricRemoved(MetricRegistry.java:568)
        at app//com.codahale.metrics.MetricRegistry.remove(MetricRegistry.java:362)
        at app//com.codahale.metrics.MetricRegistry.removeMatching(MetricRegistry.java:376)
        at app//org.apache.solr.metrics.SolrMetricManager.unregisterGauges(SolrMetricManager.java:836)
        at app//org.apache.solr.metrics.SolrCoreMetricManager.close(SolrCoreMetricManager.java:160)
        at app//org.apache.solr.core.SolrCore.doClose(SolrCore.java:1769)
        at app//org.apache.solr.core.SolrCore.close(SolrCore.java:1755)
        at app//org.apache.solr.core.CoreContainer.registerCore(CoreContainer.java:1470)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:2020)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1977)
        at app//org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$2(CoreAdminOperation.java:136)
        at app//org.apache.solr.handler.admin.CoreAdminOperation$$Lambda$512/0x0000000147dc4440.execute(Unknown Source)
        at app//org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:406)
        at app//org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:365)
        at app//org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:228)
        at app//org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:224)
        at app//org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:922)
        at app//org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:870)
        at app//org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:539)
        at app//org.apache.solr.servlet.SolrDispatchFilter.dispatch(SolrDispatchFilter.java:248)
        at app//org.apache.solr.servlet.SolrDispatchFilter.lambda$doFilter$0(SolrDispatchFilter.java:215)
        at app//org.apache.solr.servlet.SolrDispatchFilter$$Lambda$751/0x0000000148585d60.run(Unknown Source)
        at app//org.apache.solr.servlet.ServletUtils.traceHttpRequestExecution2(ServletUtils.java:241)
        at app//org.apache.solr.servlet.ServletUtils.rateLimitRequest(ServletUtils.java:211)
        at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:209)
        at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:192)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.apache.solr.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:187)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1384)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1306)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at app//org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:301)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at app//org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:790)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at app//org.eclipse.jetty.server.Server.handle(Server.java:563)
        at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at app//org.eclipse.jetty.server.HttpChannel$$Lambda$749/0x0000000148ab3040.dispatch(Unknown Source)
        at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at app//org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:457)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:969)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.doRunJob(QueuedThreadPool.java:1194)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1149)
        at java.base@11.0.15/java.lang.Thread.run(Thread.java:829)
   4) Thread[id=220, name=searcherExecutor-169-thread-1, state=WAITING, group=TGRP-GCSIncrementalBackupTest]
        at java.base@11.0.15/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.15/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.15/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.15/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.15/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.15/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.15/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.15/java.lang.Thread.run(Thread.java:829)
   5) Thread[id=56, name=qtp1436107811-56, state=WAITING, group=TGRP-GCSIncrementalBackupTest]
        at java.base@11.0.15/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.15/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.15/java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:885)
        at java.base@11.0.15/java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireShared(AbstractQueuedSynchronizer.java:1009)
        at java.base@11.0.15/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireShared(AbstractQueuedSynchronizer.java:1324)
        at java.base@11.0.15/java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.lock(ReentrantReadWriteLock.java:738)
        at java.management@11.0.15/com.sun.jmx.mbeanserver.Repository.retrieve(Repository.java:484)
        at java.management@11.0.15/com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.getMBean(DefaultMBeanServerInterceptor.java:1077)
        at java.management@11.0.15/com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.getAttribute(DefaultMBeanServerInterceptor.java:637)
        at java.management@11.0.15/com.sun.jmx.mbeanserver.JmxMBeanServer.getAttribute(JmxMBeanServer.java:678)
        at java.management@11.0.15/javax.management.AttributeValueExp.getAttribute(AttributeValueExp.java:162)
        at java.management@11.0.15/javax.management.AttributeValueExp.apply(AttributeValueExp.java:104)
        at java.management@11.0.15/javax.management.BinaryRelQueryExp.apply(BinaryRelQueryExp.java:109)
        at java.management@11.0.15/com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.filterListOfObjectInstances(DefaultMBeanServerInterceptor.java:1599)
        at java.management@11.0.15/com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.queryMBeans(DefaultMBeanServerInterceptor.java:495)
        at java.management@11.0.15/com.sun.jmx.mbeanserver.JmxMBeanServer.queryMBeans(JmxMBeanServer.java:592)
        at app//org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter$JmxListener.unregisterMBean(JmxMetricsReporter.java:567)
        at app//org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter$JmxListener.onGaugeRemoved(JmxMetricsReporter.java:602)
        at app//com.codahale.metrics.MetricRegistry.notifyListenerOfRemovedMetric(MetricRegistry.java:574)
        at app//com.codahale.metrics.MetricRegistry.onMetricRemoved(MetricRegistry.java:568)
        at app//com.codahale.metrics.MetricRegistry.remove(MetricRegistry.java:362)
        at app//com.codahale.metrics.MetricRegistry.removeMatching(MetricRegistry.java:376)
        at app//org.apache.solr.metrics.SolrMetricManager.unregisterGauges(SolrMetricManager.java:836)
        at app//org.apache.solr.metrics.SolrCoreMetricManager.close(SolrCoreMetricManager.java:160)
        at app//org.apache.solr.core.SolrCore.doClose(SolrCore.java:1769)
        at app//org.apache.solr.core.SolrCore.close(SolrCore.java:1755)
        at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1223)
        at app//org.apache.solr.core.SolrCore.reload(SolrCore.java:789)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:2009)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1977)
        at app//org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$2(CoreAdminOperation.java:136)
        at app//org.apache.solr.handler.admin.CoreAdminOperation$$Lambda$512/0x0000000147dc4440.execute(Unknown Source)
        at app//org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:406)
        at app//org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:365)
        at app//org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:228)
        at app//org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:224)
        at app//org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:922)
        at app//org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:870)
        at app//org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:539)
        at app//org.apache.solr.servlet.SolrDispatchFilter.dispatch(SolrDispatchFilter.java:248)
        at app//org.apache.solr.servlet.SolrDispatchFilter.lambda$doFilter$0(SolrDispatchFilter.java:215)
        at app//org.apache.solr.servlet.SolrDispatchFilter$$Lambda$751/0x0000000148585d60.run(Unknown Source)
        at app//org.apache.solr.servlet.ServletUtils.traceHttpRequestExecution2(ServletUtils.java:241)
        at app//org.apache.solr.servlet.ServletUtils.rateLimitRequest(ServletUtils.java:211)
        at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:209)
        at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:192)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.apache.solr.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:187)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1384)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1306)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at app//org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:301)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at app//org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:790)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at app//org.eclipse.jetty.server.Server.handle(Server.java:563)
        at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at app//org.eclipse.jetty.server.HttpChannel$$Lambda$749/0x0000000148ab3040.dispatch(Unknown Source)
        at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at app//org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:457)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:969)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.doRunJob(QueuedThreadPool.java:1194)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1149)
        at java.base@11.0.15/java.lang.Thread.run(Thread.java:829)
   6) Thread[id=242, name=searcherExecutor-171-thread-1, state=WAITING, group=TGRP-GCSIncrementalBackupTest]
        at java.base@11.0.15/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.15/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.15/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.15/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.15/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.15/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.15/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.15/java.lang.Thread.run(Thread.java:829)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 6 threads leaked from SUITE scope at org.apache.solr.gcs.GCSIncrementalBackupTest: 
   1) Thread[id=57, name=qtp1436107811-57, state=RUNNABLE, group=TGRP-GCSIncrementalBackupTest]
        at java.base@11.0.15/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.15/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.15/java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:885)
        at java.base@11.0.15/java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireShared(AbstractQueuedSynchronizer.java:1009)
        at java.base@11.0.15/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireShared(AbstractQueuedSynchronizer.java:1324)
        at java.base@11.0.15/java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.lock(ReentrantReadWriteLock.java:738)
        at java.management@11.0.15/com.sun.jmx.mbeanserver.Repository.query(Repository.java:525)
        at java.management@11.0.15/com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.queryMBeansImpl(DefaultMBeanServerInterceptor.java:507)
        at java.management@11.0.15/com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.queryMBeans(DefaultMBeanServerInterceptor.java:476)
        at java.management@11.0.15/com.sun.jmx.mbeanserver.JmxMBeanServer.queryMBeans(JmxMBeanServer.java:592)
        at app//org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter$JmxListener.unregisterMBean(JmxMetricsReporter.java:567)
        at app//org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter$JmxListener.onGaugeRemoved(JmxMetricsReporter.java:602)
        at app//com.codahale.metrics.MetricRegistry.notifyListenerOfRemovedMetric(MetricRegistry.java:574)
        at app//com.codahale.metrics.MetricRegistry.onMetricRemoved(MetricRegistry.java:568)
        at app//com.codahale.metrics.MetricRegistry.remove(MetricRegistry.java:362)
        at app//com.codahale.metrics.MetricRegistry.removeMatching(MetricRegistry.java:376)
        at app//org.apache.solr.metrics.SolrMetricManager.unregisterGauges(SolrMetricManager.java:836)
        at app//org.apache.solr.metrics.SolrCoreMetricManager.close(SolrCoreMetricManager.java:160)
        at app//org.apache.solr.core.SolrCore.doClose(SolrCore.java:1769)
        at app//org.apache.solr.core.SolrCore.close(SolrCore.java:1755)
        at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1223)
        at app//org.apache.solr.core.SolrCore.reload(SolrCore.java:789)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:2009)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1977)
        at app//org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$2(CoreAdminOperation.java:136)
        at app//org.apache.solr.handler.admin.CoreAdminOperation$$Lambda$512/0x0000000147dc4440.execute(Unknown Source)
        at app//org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:406)
        at app//org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:365)
        at app//org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:228)
        at app//org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:224)
        at app//org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:922)
        at app//org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:870)
        at app//org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:539)
        at app//org.apache.solr.servlet.SolrDispatchFilter.dispatch(SolrDispatchFilter.java:248)
        at app//org.apache.solr.servlet.SolrDispatchFilter.lambda$doFilter$0(SolrDispatchFilter.java:215)
        at app//org.apache.solr.servlet.SolrDispatchFilter$$Lambda$751/0x0000000148585d60.run(Unknown Source)
        at app//org.apache.solr.servlet.ServletUtils.traceHttpRequestExecution2(ServletUtils.java:241)
        at app//org.apache.solr.servlet.ServletUtils.rateLimitRequest(ServletUtils.java:211)
        at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:209)
        at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:192)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.apache.solr.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:187)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1384)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1306)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at app//org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:301)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at app//org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:790)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at app//org.eclipse.jetty.server.Server.handle(Server.java:563)
        at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at app//org.eclipse.jetty.server.HttpChannel$$Lambda$749/0x0000000148ab3040.dispatch(Unknown Source)
        at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at app//org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:457)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:969)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.doRunJob(QueuedThreadPool.java:1194)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1149)
        at java.base@11.0.15/java.lang.Thread.run(Thread.java:829)
   2) Thread[id=243, name=searcherExecutor-173-thread-1, state=WAITING, group=TGRP-GCSIncrementalBackupTest]
        at java.base@11.0.15/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.15/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.15/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.15/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.15/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.15/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.15/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.15/java.lang.Thread.run(Thread.java:829)
   3) Thread[id=49, name=qtp1490440899-49, state=RUNNABLE, group=TGRP-GCSIncrementalBackupTest]
        at java.base@11.0.15/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.15/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.15/java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:885)
        at java.base@11.0.15/java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireShared(AbstractQueuedSynchronizer.java:1009)
        at java.base@11.0.15/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireShared(AbstractQueuedSynchronizer.java:1324)
        at java.base@11.0.15/java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.lock(ReentrantReadWriteLock.java:738)
        at java.management@11.0.15/com.sun.jmx.mbeanserver.Repository.query(Repository.java:525)
        at java.management@11.0.15/com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.queryMBeansImpl(DefaultMBeanServerInterceptor.java:507)
        at java.management@11.0.15/com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.queryMBeans(DefaultMBeanServerInterceptor.java:476)
        at java.management@11.0.15/com.sun.jmx.mbeanserver.JmxMBeanServer.queryMBeans(JmxMBeanServer.java:592)
        at app//org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter$JmxListener.unregisterMBean(JmxMetricsReporter.java:567)
        at app//org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter$JmxListener.onGaugeRemoved(JmxMetricsReporter.java:602)
        at app//com.codahale.metrics.MetricRegistry.notifyListenerOfRemovedMetric(MetricRegistry.java:574)
        at app//com.codahale.metrics.MetricRegistry.onMetricRemoved(MetricRegistry.java:568)
        at app//com.codahale.metrics.MetricRegistry.remove(MetricRegistry.java:362)
        at app//com.codahale.metrics.MetricRegistry.removeMatching(MetricRegistry.java:376)
        at app//org.apache.solr.metrics.SolrMetricManager.unregisterGauges(SolrMetricManager.java:836)
        at app//org.apache.solr.metrics.SolrCoreMetricManager.close(SolrCoreMetricManager.java:160)
        at app//org.apache.solr.core.SolrCore.doClose(SolrCore.java:1769)
        at app//org.apache.solr.core.SolrCore.close(SolrCore.java:1755)
        at app//org.apache.solr.core.CoreContainer.registerCore(CoreContainer.java:1470)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:2020)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1977)
        at app//org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$2(CoreAdminOperation.java:136)
        at app//org.apache.solr.handler.admin.CoreAdminOperation$$Lambda$512/0x0000000147dc4440.execute(Unknown Source)
        at app//org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:406)
        at app//org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:365)
        at app//org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:228)
        at app//org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:224)
        at app//org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:922)
        at app//org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:870)
        at app//org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:539)
        at app//org.apache.solr.servlet.SolrDispatchFilter.dispatch(SolrDispatchFilter.java:248)
        at app//org.apache.solr.servlet.SolrDispatchFilter.lambda$doFilter$0(SolrDispatchFilter.java:215)
        at app//org.apache.solr.servlet.SolrDispatchFilter$$Lambda$751/0x0000000148585d60.run(Unknown Source)
        at app//org.apache.solr.servlet.ServletUtils.traceHttpRequestExecution2(ServletUtils.java:241)
        at app//org.apache.solr.servlet.ServletUtils.rateLimitRequest(ServletUtils.java:211)
        at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:209)
        at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:192)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.apache.solr.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:187)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1384)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1306)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at app//org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:301)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at app//org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:790)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at app//org.eclipse.jetty.server.Server.handle(Server.java:563)
        at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at app//org.eclipse.jetty.server.HttpChannel$$Lambda$749/0x0000000148ab3040.dispatch(Unknown Source)
        at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at app//org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:457)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:969)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.doRunJob(QueuedThreadPool.java:1194)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1149)
        at java.base@11.0.15/java.lang.Thread.run(Thread.java:829)
   4) Thread[id=220, name=searcherExecutor-169-thread-1, state=WAITING, group=TGRP-GCSIncrementalBackupTest]
        at java.base@11.0.15/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.15/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.15/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.15/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.15/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.15/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.15/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.15/java.lang.Thread.run(Thread.java:829)
   5) Thread[id=56, name=qtp1436107811-56, state=WAITING, group=TGRP-GCSIncrementalBackupTest]
        at java.base@11.0.15/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.15/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.15/java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:885)
        at java.base@11.0.15/java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireShared(AbstractQueuedSynchronizer.java:1009)
        at java.base@11.0.15/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireShared(AbstractQueuedSynchronizer.java:1324)
        at java.base@11.0.15/java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.lock(ReentrantReadWriteLock.java:738)
        at java.management@11.0.15/com.sun.jmx.mbeanserver.Repository.retrieve(Repository.java:484)
        at java.management@11.0.15/com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.getMBean(DefaultMBeanServerInterceptor.java:1077)
        at java.management@11.0.15/com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.getAttribute(DefaultMBeanServerInterceptor.java:637)
        at java.management@11.0.15/com.sun.jmx.mbeanserver.JmxMBeanServer.getAttribute(JmxMBeanServer.java:678)
        at java.management@11.0.15/javax.management.AttributeValueExp.getAttribute(AttributeValueExp.java:162)
        at java.management@11.0.15/javax.management.AttributeValueExp.apply(AttributeValueExp.java:104)
        at java.management@11.0.15/javax.management.BinaryRelQueryExp.apply(BinaryRelQueryExp.java:109)
        at java.management@11.0.15/com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.filterListOfObjectInstances(DefaultMBeanServerInterceptor.java:1599)
        at java.management@11.0.15/com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.queryMBeans(DefaultMBeanServerInterceptor.java:495)
        at java.management@11.0.15/com.sun.jmx.mbeanserver.JmxMBeanServer.queryMBeans(JmxMBeanServer.java:592)
        at app//org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter$JmxListener.unregisterMBean(JmxMetricsReporter.java:567)
        at app//org.apache.solr.metrics.reporters.jmx.JmxMetricsReporter$JmxListener.onGaugeRemoved(JmxMetricsReporter.java:602)
        at app//com.codahale.metrics.MetricRegistry.notifyListenerOfRemovedMetric(MetricRegistry.java:574)
        at app//com.codahale.metrics.MetricRegistry.onMetricRemoved(MetricRegistry.java:568)
        at app//com.codahale.metrics.MetricRegistry.remove(MetricRegistry.java:362)
        at app//com.codahale.metrics.MetricRegistry.removeMatching(MetricRegistry.java:376)
        at app//org.apache.solr.metrics.SolrMetricManager.unregisterGauges(SolrMetricManager.java:836)
        at app//org.apache.solr.metrics.SolrCoreMetricManager.close(SolrCoreMetricManager.java:160)
        at app//org.apache.solr.core.SolrCore.doClose(SolrCore.java:1769)
        at app//org.apache.solr.core.SolrCore.close(SolrCore.java:1755)
        at app//org.apache.solr.core.SolrCore.<init>(SolrCore.java:1223)
        at app//org.apache.solr.core.SolrCore.reload(SolrCore.java:789)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:2009)
        at app//org.apache.solr.core.CoreContainer.reload(CoreContainer.java:1977)
        at app//org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$2(CoreAdminOperation.java:136)
        at app//org.apache.solr.handler.admin.CoreAdminOperation$$Lambda$512/0x0000000147dc4440.execute(Unknown Source)
        at app//org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:406)
        at app//org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:365)
        at app//org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:228)
        at app//org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:224)
        at app//org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:922)
        at app//org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:870)
        at app//org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:539)
        at app//org.apache.solr.servlet.SolrDispatchFilter.dispatch(SolrDispatchFilter.java:248)
        at app//org.apache.solr.servlet.SolrDispatchFilter.lambda$doFilter$0(SolrDispatchFilter.java:215)
        at app//org.apache.solr.servlet.SolrDispatchFilter$$Lambda$751/0x0000000148585d60.run(Unknown Source)
        at app//org.apache.solr.servlet.ServletUtils.traceHttpRequestExecution2(ServletUtils.java:241)
        at app//org.apache.solr.servlet.ServletUtils.rateLimitRequest(ServletUtils.java:211)
        at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:209)
        at app//org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:192)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.apache.solr.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:187)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1384)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1306)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at app//org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:301)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at app//org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:790)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at app//org.eclipse.jetty.server.Server.handle(Server.java:563)
        at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at app//org.eclipse.jetty.server.HttpChannel$$Lambda$749/0x0000000148ab3040.dispatch(Unknown Source)
        at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at app//org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:457)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:969)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.doRunJob(QueuedThreadPool.java:1194)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1149)
        at java.base@11.0.15/java.lang.Thread.run(Thread.java:829)
   6) Thread[id=242, name=searcherExecutor-171-thread-1, state=WAITING, group=TGRP-GCSIncrementalBackupTest]
        at java.base@11.0.15/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11.0.15/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11.0.15/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11.0.15/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11.0.15/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11.0.15/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11.0.15/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11.0.15/java.lang.Thread.run(Thread.java:829)
	at __randomizedtesting.SeedInfo.seed([80F9D56CC1404BF]:0)


FAILED:  org.apache.solr.security.hadoop.SaslZkACLProviderTest.testSaslZkACLProvider

Error Message:
org.apache.solr.common.SolrException: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper localhost:59303/solr within 15000 ms

Stack Trace:
org.apache.solr.common.SolrException: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper localhost:59303/solr within 15000 ms
	at __randomizedtesting.SeedInfo.seed([80F9D56CC1404BF:CFBCB8E5550A7B1]:0)
	at app//org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:209)
	at app//org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:111)
	at app//org.apache.solr.security.hadoop.SaslZkACLProviderTest$SolrZkClientWithACLs.<init>(SaslZkACLProviderTest.java:182)
	at app//org.apache.solr.security.hadoop.SaslZkACLProviderTest.setupZNodes(SaslZkACLProviderTest.java:111)
	at app//org.apache.solr.security.hadoop.SaslZkACLProviderTest.setUp(SaslZkACLProviderTest.java:102)
	at java.base@11.0.15/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base@11.0.15/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base@11.0.15/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base@11.0.15/java.lang.reflect.Method.invoke(Method.java:566)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:980)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
	at java.base@11.0.15/java.lang.Thread.run(Thread.java:829)
Caused by: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper localhost:59303/solr within 15000 ms
	at org.apache.solr.common.cloud.ConnectionManager.waitForConnected(ConnectionManager.java:296)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:200)
	... 49 more