You are viewing a plain text version of this content. The canonical link for it is here.
Posted to builds@solr.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2023/04/03 04:59:28 UTC

[JENKINS] Solr-main-MacOSX (64bit/hotspot/jdk-16.0.2) - Build # 2594 - Still Unstable!

Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/2594/
Java: 64bit/hotspot/jdk-16.0.2 -XX:+UseCompressedOops -XX:+UseG1GC

2 tests failed.
FAILED:  org.apache.solr.cloud.ReplaceNodeTest.testGoodSpreadDuringAssignWithNoTarget

Error Message:
java.lang.AssertionError: There should be no more replicas on the sourceNode after a replaceNode request. expected:<[]> but was:<[core_node11:{
  "core":"replacenodetest_coll_shard1_replica_n5",
  "leader":"true",
  "node_name":"127.0.0.1:53395_solr",
  "base_url":"https://127.0.0.1:53395/solr",
  "state":"active",
  "type":"NRT",
  "force_set_state":"false"}, core_node7:{
  "core":"replacenodetest_coll_shard2_replica_n1",
  "leader":"true",
  "node_name":"127.0.0.1:53395_solr",
  "base_url":"https://127.0.0.1:53395/solr",
  "state":"active",
  "type":"NRT",
  "force_set_state":"false"}, core_node8:{
  "core":"replacenodetest_coll_shard3_replica_n2",
  "leader":"true",
  "node_name":"127.0.0.1:53395_solr",
  "base_url":"https://127.0.0.1:53395/solr",
  "state":"active",
  "type":"NRT",
  "force_set_state":"false"}]>

Stack Trace:
java.lang.AssertionError: There should be no more replicas on the sourceNode after a replaceNode request. expected:<[]> but was:<[core_node11:{
  "core":"replacenodetest_coll_shard1_replica_n5",
  "leader":"true",
  "node_name":"127.0.0.1:53395_solr",
  "base_url":"https://127.0.0.1:53395/solr",
  "state":"active",
  "type":"NRT",
  "force_set_state":"false"}, core_node7:{
  "core":"replacenodetest_coll_shard2_replica_n1",
  "leader":"true",
  "node_name":"127.0.0.1:53395_solr",
  "base_url":"https://127.0.0.1:53395/solr",
  "state":"active",
  "type":"NRT",
  "force_set_state":"false"}, core_node8:{
  "core":"replacenodetest_coll_shard3_replica_n2",
  "leader":"true",
  "node_name":"127.0.0.1:53395_solr",
  "base_url":"https://127.0.0.1:53395/solr",
  "state":"active",
  "type":"NRT",
  "force_set_state":"false"}]>
	at __randomizedtesting.SeedInfo.seed([74DCE7154AA24B6:29261A4B61C9F5C2]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.junit.Assert.failNotEquals(Assert.java:835)
	at org.junit.Assert.assertEquals(Assert.java:120)
	at org.apache.solr.cloud.ReplaceNodeTest.testGoodSpreadDuringAssignWithNoTarget(ReplaceNodeTest.java:274)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:78)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:567)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
	at java.base/java.lang.Thread.run(Thread.java:831)


FAILED:  org.apache.solr.cloud.ReplaceNodeTest.test

Error Message:
java.lang.AssertionError: Expecting no cores but found some: {replacenodetest_coll_shard1_replica_n21={name=replacenodetest_coll_shard1_replica_n21, instanceDir=/Users/jenkins/workspace/Solr-main-MacOSX/solr/core/build/tmp/tests-tmp/solr.cloud.ReplaceNodeTest_74DCE7154AA24B6-001/tempDir-002/node5/replacenodetest_coll_shard1_replica_n21, dataDir=/Users/jenkins/workspace/Solr-main-MacOSX/solr/core/build/tmp/tests-tmp/solr.cloud.ReplaceNodeTest_74DCE7154AA24B6-001/tempDir-002/node5/replacenodetest_coll_shard1_replica_n21/data/, config=solrconfig.xml, schema=schema.xml, startTime=Mon Apr 03 10:25:09 BDT 2023, uptime=33436, lastPublished=active, configVersion=0, cloud={collection=replacenodetest_coll, shard=shard1, replica=core_node22, replicaType=NRT}, index={numDocs=0, maxDoc=0, deletedDocs=0, version=2, segmentCount=0, current=true, hasDeletions=false, directory=org.apache.lucene.tests.store.MockDirectoryWrapper:MockDirectoryWrapper(ByteBuffersDirectory@f5150f4 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@17169c01), segmentsFile=segments_1, segmentsFileSizeInBytes=69, userData={}, sizeInBytes=69, size=69 bytes}}, replacenodetest_coll_shard3_replica_n23={}} expected:<0> but was:<2>

Stack Trace:
java.lang.AssertionError: Expecting no cores but found some: {replacenodetest_coll_shard1_replica_n21={name=replacenodetest_coll_shard1_replica_n21, instanceDir=/Users/jenkins/workspace/Solr-main-MacOSX/solr/core/build/tmp/tests-tmp/solr.cloud.ReplaceNodeTest_74DCE7154AA24B6-001/tempDir-002/node5/replacenodetest_coll_shard1_replica_n21, dataDir=/Users/jenkins/workspace/Solr-main-MacOSX/solr/core/build/tmp/tests-tmp/solr.cloud.ReplaceNodeTest_74DCE7154AA24B6-001/tempDir-002/node5/replacenodetest_coll_shard1_replica_n21/data/, config=solrconfig.xml, schema=schema.xml, startTime=Mon Apr 03 10:25:09 BDT 2023, uptime=33436, lastPublished=active, configVersion=0, cloud={collection=replacenodetest_coll, shard=shard1, replica=core_node22, replicaType=NRT}, index={numDocs=0, maxDoc=0, deletedDocs=0, version=2, segmentCount=0, current=true, hasDeletions=false, directory=org.apache.lucene.tests.store.MockDirectoryWrapper:MockDirectoryWrapper(ByteBuffersDirectory@f5150f4 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@17169c01), segmentsFile=segments_1, segmentsFileSizeInBytes=69, userData={}, sizeInBytes=69, size=69 bytes}}, replacenodetest_coll_shard3_replica_n23={}} expected:<0> but was:<2>
	at __randomizedtesting.SeedInfo.seed([74DCE7154AA24B6:8F19F1ABFA56494E]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.junit.Assert.failNotEquals(Assert.java:835)
	at org.junit.Assert.assertEquals(Assert.java:647)
	at org.apache.solr.cloud.ReplaceNodeTest.test(ReplaceNodeTest.java:147)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:78)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:567)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
	at java.base/java.lang.Thread.run(Thread.java:831)

[JENKINS] Solr-main-MacOSX (64bit/hotspot/jdk-11.0.15) - Build # 2604 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/2604/
Java: 64bit/hotspot/jdk-11.0.15 -XX:-UseCompressedOops -XX:+UseConcMarkSweepGC

4 tests failed.
FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
java.lang.AssertionError: Didn't see all replicas for shard shard1 in collection1 come up within 30000 ms! ClusterState: {
  "control_collection":{
    "pullReplicas":"0",
    "configName":"conf1",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{"core_node2":{
            "core":"control_collection_shard1_replica_n1",
            "leader":"true",
            "node_name":"127.0.0.1:53276_",
            "base_url":"http://127.0.0.1:53276",
            "state":"active",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"},
  "collection1":{
    "pullReplicas":"0",
    "configName":"conf1",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{
          "core_node2":{
            "core":"collection1_shard1_replica_n1",
            "leader":"true",
            "node_name":"127.0.0.1:53303_",
            "base_url":"http://127.0.0.1:53303",
            "state":"active",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node4":{
            "core":"collection1_shard1_replica_n3",
            "node_name":"127.0.0.1:53312_",
            "base_url":"http://127.0.0.1:53312",
            "state":"active",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"}}

Stack Trace:
java.lang.AssertionError: Didn't see all replicas for shard shard1 in collection1 come up within 30000 ms! ClusterState: {
  "control_collection":{
    "pullReplicas":"0",
    "configName":"conf1",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{"core_node2":{
            "core":"control_collection_shard1_replica_n1",
            "leader":"true",
            "node_name":"127.0.0.1:53276_",
            "base_url":"http://127.0.0.1:53276",
            "state":"active",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"},
  "collection1":{
    "pullReplicas":"0",
    "configName":"conf1",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{
          "core_node2":{
            "core":"collection1_shard1_replica_n1",
            "leader":"true",
            "node_name":"127.0.0.1:53303_",
            "base_url":"http://127.0.0.1:53303",
            "state":"active",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node4":{
            "core":"collection1_shard1_replica_n3",
            "node_name":"127.0.0.1:53312_",
            "base_url":"http://127.0.0.1:53312",
            "state":"active",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"}}
	at __randomizedtesting.SeedInfo.seed([BEB894D975CD3FFD:36ECAB03DB315205]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.ensureAllReplicasAreActive(AbstractFullDistribZkTestBase.java:2531)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:123)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1163)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1134)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
	at java.base/java.lang.Thread.run(Thread.java:829)


FAILED:  org.apache.solr.s3.S3BackupRepositoryTest.testCopyFiles

Error Message:
software.amazon.awssdk.core.exception.SdkClientException: Unable to execute HTTP request: Read timed out

Stack Trace:
software.amazon.awssdk.core.exception.SdkClientException: Unable to execute HTTP request: Read timed out
	at __randomizedtesting.SeedInfo.seed([BEB894D975CD3FFD:7064A0ED5F73128D]:0)
	at app//software.amazon.awssdk.core.exception.SdkClientException$BuilderImpl.build(SdkClientException.java:111)
	at app//software.amazon.awssdk.core.exception.SdkClientException.create(SdkClientException.java:47)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.utils.RetryableStageHelper.setLastException(RetryableStageHelper.java:223)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.RetryableStage.execute(RetryableStage.java:83)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.RetryableStage.execute(RetryableStage.java:36)
	at app//software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206)
	at app//software.amazon.awssdk.core.internal.http.StreamManagingStage.execute(StreamManagingStage.java:56)
	at app//software.amazon.awssdk.core.internal.http.StreamManagingStage.execute(StreamManagingStage.java:36)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallTimeoutTrackingStage.executeWithTimer(ApiCallTimeoutTrackingStage.java:80)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallTimeoutTrackingStage.execute(ApiCallTimeoutTrackingStage.java:60)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallTimeoutTrackingStage.execute(ApiCallTimeoutTrackingStage.java:42)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallMetricCollectionStage.execute(ApiCallMetricCollectionStage.java:48)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallMetricCollectionStage.execute(ApiCallMetricCollectionStage.java:31)
	at app//software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206)
	at app//software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.ExecutionFailureExceptionReportingStage.execute(ExecutionFailureExceptionReportingStage.java:37)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.ExecutionFailureExceptionReportingStage.execute(ExecutionFailureExceptionReportingStage.java:26)
	at app//software.amazon.awssdk.core.internal.http.AmazonSyncHttpClient$RequestExecutionBuilderImpl.execute(AmazonSyncHttpClient.java:193)
	at app//software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.invoke(BaseSyncClientHandler.java:103)
	at app//software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.doExecute(BaseSyncClientHandler.java:171)
	at app//software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.lambda$execute$1(BaseSyncClientHandler.java:82)
	at app//software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.measureApiCallSuccess(BaseSyncClientHandler.java:179)
	at app//software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.execute(BaseSyncClientHandler.java:76)
	at app//software.amazon.awssdk.core.client.handler.SdkSyncClientHandler.execute(SdkSyncClientHandler.java:45)
	at app//software.amazon.awssdk.awscore.client.handler.AwsSyncClientHandler.execute(AwsSyncClientHandler.java:56)
	at app//software.amazon.awssdk.services.s3.DefaultS3Client.putObject(DefaultS3Client.java:9326)
	at app//software.amazon.awssdk.services.s3.S3Client.putObject(S3Client.java:17280)
	at app//org.apache.solr.s3.S3BackupRepositoryTest.pushObject(S3BackupRepositoryTest.java:328)
	at app//org.apache.solr.s3.S3BackupRepositoryTest.doTestCopyFileTo(S3BackupRepositoryTest.java:214)
	at app//org.apache.solr.s3.S3BackupRepositoryTest.testCopyFiles(S3BackupRepositoryTest.java:173)
	at java.base@11.0.15/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base@11.0.15/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base@11.0.15/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base@11.0.15/java.lang.reflect.Method.invoke(Method.java:566)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at app//org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:54)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at app//com.adobe.testing.s3mock.junit4.S3MockRule$1.evaluate(S3MockRule.java:66)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
	at java.base@11.0.15/java.lang.Thread.run(Thread.java:829)
	Suppressed: software.amazon.awssdk.core.exception.SdkClientException: Request attempt 1 failure: Unable to execute HTTP request: Read timed out
	Suppressed: software.amazon.awssdk.core.exception.SdkClientException: Request attempt 2 failure: Unable to execute HTTP request: Read timed out
	Suppressed: software.amazon.awssdk.core.exception.SdkClientException: Request attempt 3 failure: Unable to execute HTTP request: Read timed out
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.base/java.net.SocketInputStream.socketRead0(Native Method)
	at java.base/java.net.SocketInputStream.socketRead(SocketInputStream.java:115)
	at java.base/java.net.SocketInputStream.read(SocketInputStream.java:168)
	at java.base/java.net.SocketInputStream.read(SocketInputStream.java:140)
	at java.base/java.io.BufferedInputStream.fill(BufferedInputStream.java:252)
	at java.base/java.io.BufferedInputStream.read1(BufferedInputStream.java:292)
	at java.base/java.io.BufferedInputStream.read(BufferedInputStream.java:351)
	at java.base/sun.net.www.http.HttpClient.parseHTTPHeader(HttpClient.java:754)
	at java.base/sun.net.www.http.HttpClient.parseHTTP(HttpClient.java:689)
	at java.base/sun.net.www.protocol.http.HttpURLConnection.getInputStream0(HttpURLConnection.java:1615)
	at java.base/sun.net.www.protocol.http.HttpURLConnection.getInputStream(HttpURLConnection.java:1520)
	at java.base/java.net.HttpURLConnection.getResponseCode(HttpURLConnection.java:527)
	at software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient$RequestCallable.getResponseCodeSafely(UrlConnectionHttpClient.java:419)
	at software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient$RequestCallable.call(UrlConnectionHttpClient.java:302)
	at software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient$RequestCallable.call(UrlConnectionHttpClient.java:269)
	at software.amazon.awssdk.core.internal.util.MetricUtils.measureDurationUnsafe(MetricUtils.java:63)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.MakeHttpRequestStage.executeHttpRequest(MakeHttpRequestStage.java:77)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.MakeHttpRequestStage.execute(MakeHttpRequestStage.java:56)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.MakeHttpRequestStage.execute(MakeHttpRequestStage.java:39)
	at software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206)
	at software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206)
	at software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206)
	at software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallAttemptTimeoutTrackingStage.execute(ApiCallAttemptTimeoutTrackingStage.java:73)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallAttemptTimeoutTrackingStage.execute(ApiCallAttemptTimeoutTrackingStage.java:42)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.TimeoutExceptionHandlingStage.execute(TimeoutExceptionHandlingStage.java:78)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.TimeoutExceptionHandlingStage.execute(TimeoutExceptionHandlingStage.java:40)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallAttemptMetricCollectionStage.execute(ApiCallAttemptMetricCollectionStage.java:50)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallAttemptMetricCollectionStage.execute(ApiCallAttemptMetricCollectionStage.java:36)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.RetryableStage.execute(RetryableStage.java:81)
	... 75 more


FAILED:  org.apache.solr.s3.S3BackupRepositoryTest.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.s3.S3BackupRepositoryTest: 
   1) Thread[id=47, name=qtp1835695775-47, state=BLOCKED, group=TGRP-S3BackupRepositoryTest]
        at app//com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:133)
        at app//com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at app//com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at java.base@11.0.15/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at java.base@11.0.15/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at java.base@11.0.15/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.base@11.0.15/java.lang.reflect.Method.invoke(Method.java:566)
        at app//org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at app//org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at app//org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at app//org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at app//org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at app//org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at app//org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at app//org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at app//org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at app//org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at app//javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at app//org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at app//javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at app//org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at app//org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at app//com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at app//org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at app//org.eclipse.jetty.server.Server.handle(Server.java:563)
        at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at app//org.eclipse.jetty.server.HttpChannel$$Lambda$1319/0x000000014819ac40.dispatch(Unknown Source)
        at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at app//org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at app//org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at app//org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at app//org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at app//org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.runTask(AdaptiveExecutionStrategy.java:416)
        at app//org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.consumeTask(AdaptiveExecutionStrategy.java:385)
        at app//org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:272)
        at app//org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.lambda$new$0(AdaptiveExecutionStrategy.java:140)
        at app//org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy$$Lambda$968/0x0000000146e9f840.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:411)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base@11.0.15/java.lang.Thread.run(Thread.java:829)
   2) Thread[id=58, name=qtp1835695775-58, state=RUNNABLE, group=TGRP-S3BackupRepositoryTest]
        at java.base@11.0.15/sun.nio.ch.FileDispatcherImpl.read0(Native Method)
        at java.base@11.0.15/sun.nio.ch.FileDispatcherImpl.read(FileDispatcherImpl.java:48)
        at java.base@11.0.15/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:276)
        at java.base@11.0.15/sun.nio.ch.IOUtil.read(IOUtil.java:245)
        at java.base@11.0.15/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:223)
        at java.base@11.0.15/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:65)
        at java.base@11.0.15/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:107)
        at java.base@11.0.15/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:101)
        at java.base@11.0.15/java.io.InputStream.read(InputStream.java:205)
        at java.base@11.0.15/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:81)
        at app//com.adobe.testing.s3mock.util.AwsChunkedDecodingInputStream.read(AwsChunkedDecodingInputStream.java:91)
        at java.base@11.0.15/java.io.InputStream.read(InputStream.java:280)
        at java.base@11.0.15/java.io.InputStream.read(InputStream.java:205)
        at app//com.adobe.testing.s3mock.store.ObjectStore.inputStreamToFile(ObjectStore.java:370)
        at app//com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:135)
        at app//com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at app//com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at jdk.internal.reflect.GeneratedMethodAccessor120.invoke(Unknown Source)
        at java.base@11.0.15/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.base@11.0.15/java.lang.reflect.Method.invoke(Method.java:566)
        at app//org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at app//org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at app//org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at app//org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at app//org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at app//org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at app//org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at app//org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at app//org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at app//org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at app//javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at app//org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at app//javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at app//org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at app//org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at app//com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at app//org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at app//org.eclipse.jetty.server.Server.handle(Server.java:563)
        at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at app//org.eclipse.jetty.server.HttpChannel$$Lambda$1319/0x000000014819ac40.dispatch(Unknown Source)
        at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at app//org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at app//org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at app//org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at app//org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base@11.0.15/java.lang.Thread.run(Thread.java:829)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.s3.S3BackupRepositoryTest: 
   1) Thread[id=47, name=qtp1835695775-47, state=BLOCKED, group=TGRP-S3BackupRepositoryTest]
        at app//com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:133)
        at app//com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at app//com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at java.base@11.0.15/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at java.base@11.0.15/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at java.base@11.0.15/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.base@11.0.15/java.lang.reflect.Method.invoke(Method.java:566)
        at app//org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at app//org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at app//org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at app//org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at app//org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at app//org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at app//org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at app//org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at app//org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at app//org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at app//javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at app//org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at app//javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at app//org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at app//org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at app//com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at app//org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at app//org.eclipse.jetty.server.Server.handle(Server.java:563)
        at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at app//org.eclipse.jetty.server.HttpChannel$$Lambda$1319/0x000000014819ac40.dispatch(Unknown Source)
        at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at app//org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at app//org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at app//org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at app//org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at app//org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.runTask(AdaptiveExecutionStrategy.java:416)
        at app//org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.consumeTask(AdaptiveExecutionStrategy.java:385)
        at app//org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:272)
        at app//org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.lambda$new$0(AdaptiveExecutionStrategy.java:140)
        at app//org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy$$Lambda$968/0x0000000146e9f840.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:411)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base@11.0.15/java.lang.Thread.run(Thread.java:829)
   2) Thread[id=58, name=qtp1835695775-58, state=RUNNABLE, group=TGRP-S3BackupRepositoryTest]
        at java.base@11.0.15/sun.nio.ch.FileDispatcherImpl.read0(Native Method)
        at java.base@11.0.15/sun.nio.ch.FileDispatcherImpl.read(FileDispatcherImpl.java:48)
        at java.base@11.0.15/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:276)
        at java.base@11.0.15/sun.nio.ch.IOUtil.read(IOUtil.java:245)
        at java.base@11.0.15/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:223)
        at java.base@11.0.15/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:65)
        at java.base@11.0.15/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:107)
        at java.base@11.0.15/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:101)
        at java.base@11.0.15/java.io.InputStream.read(InputStream.java:205)
        at java.base@11.0.15/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:81)
        at app//com.adobe.testing.s3mock.util.AwsChunkedDecodingInputStream.read(AwsChunkedDecodingInputStream.java:91)
        at java.base@11.0.15/java.io.InputStream.read(InputStream.java:280)
        at java.base@11.0.15/java.io.InputStream.read(InputStream.java:205)
        at app//com.adobe.testing.s3mock.store.ObjectStore.inputStreamToFile(ObjectStore.java:370)
        at app//com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:135)
        at app//com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at app//com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at jdk.internal.reflect.GeneratedMethodAccessor120.invoke(Unknown Source)
        at java.base@11.0.15/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.base@11.0.15/java.lang.reflect.Method.invoke(Method.java:566)
        at app//org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at app//org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at app//org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at app//org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at app//org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at app//org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at app//org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at app//org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at app//org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at app//org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at app//javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at app//org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at app//javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at app//org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at app//org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at app//com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at app//org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at app//org.eclipse.jetty.server.Server.handle(Server.java:563)
        at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at app//org.eclipse.jetty.server.HttpChannel$$Lambda$1319/0x000000014819ac40.dispatch(Unknown Source)
        at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at app//org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at app//org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at app//org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at app//org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base@11.0.15/java.lang.Thread.run(Thread.java:829)
	at __randomizedtesting.SeedInfo.seed([BEB894D975CD3FFD]:0)


FAILED:  org.apache.solr.s3.S3BackupRepositoryTest.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=47, name=qtp1835695775-47, state=BLOCKED, group=TGRP-S3BackupRepositoryTest]
        at app//com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:133)
        at app//com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at app//com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at java.base@11.0.15/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at java.base@11.0.15/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at java.base@11.0.15/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.base@11.0.15/java.lang.reflect.Method.invoke(Method.java:566)
        at app//org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at app//org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at app//org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at app//org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at app//org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at app//org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at app//org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at app//org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at app//org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at app//org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at app//javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at app//org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at app//javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at app//org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at app//org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at app//com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at app//org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at app//org.eclipse.jetty.server.Server.handle(Server.java:563)
        at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at app//org.eclipse.jetty.server.HttpChannel$$Lambda$1319/0x000000014819ac40.dispatch(Unknown Source)
        at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at app//org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at app//org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at app//org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at app//org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at app//org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.runTask(AdaptiveExecutionStrategy.java:416)
        at app//org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.consumeTask(AdaptiveExecutionStrategy.java:385)
        at app//org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:272)
        at app//org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.lambda$new$0(AdaptiveExecutionStrategy.java:140)
        at app//org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy$$Lambda$968/0x0000000146e9f840.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:411)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base@11.0.15/java.lang.Thread.run(Thread.java:829)
   2) Thread[id=58, name=qtp1835695775-58, state=RUNNABLE, group=TGRP-S3BackupRepositoryTest]
        at java.base@11.0.15/sun.nio.ch.FileDispatcherImpl.read0(Native Method)
        at java.base@11.0.15/sun.nio.ch.FileDispatcherImpl.read(FileDispatcherImpl.java:48)
        at java.base@11.0.15/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:276)
        at java.base@11.0.15/sun.nio.ch.IOUtil.read(IOUtil.java:245)
        at java.base@11.0.15/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:223)
        at java.base@11.0.15/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:65)
        at java.base@11.0.15/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:107)
        at java.base@11.0.15/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:101)
        at java.base@11.0.15/java.io.InputStream.read(InputStream.java:205)
        at java.base@11.0.15/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:81)
        at app//com.adobe.testing.s3mock.util.AwsChunkedDecodingInputStream.read(AwsChunkedDecodingInputStream.java:91)
        at java.base@11.0.15/java.io.InputStream.read(InputStream.java:280)
        at java.base@11.0.15/java.io.InputStream.read(InputStream.java:205)
        at app//com.adobe.testing.s3mock.store.ObjectStore.inputStreamToFile(ObjectStore.java:370)
        at app//com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:135)
        at app//com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at app//com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at jdk.internal.reflect.GeneratedMethodAccessor120.invoke(Unknown Source)
        at java.base@11.0.15/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.base@11.0.15/java.lang.reflect.Method.invoke(Method.java:566)
        at app//org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at app//org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at app//org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at app//org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at app//org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at app//org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at app//org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at app//org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at app//org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at app//org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at app//javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at app//org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at app//javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at app//org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at app//org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at app//com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at app//org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at app//org.eclipse.jetty.server.Server.handle(Server.java:563)
        at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at app//org.eclipse.jetty.server.HttpChannel$$Lambda$1319/0x000000014819ac40.dispatch(Unknown Source)
        at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at app//org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at app//org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at app//org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at app//org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base@11.0.15/java.lang.Thread.run(Thread.java:829)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=47, name=qtp1835695775-47, state=BLOCKED, group=TGRP-S3BackupRepositoryTest]
        at app//com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:133)
        at app//com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at app//com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at java.base@11.0.15/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at java.base@11.0.15/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at java.base@11.0.15/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.base@11.0.15/java.lang.reflect.Method.invoke(Method.java:566)
        at app//org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at app//org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at app//org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at app//org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at app//org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at app//org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at app//org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at app//org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at app//org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at app//org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at app//javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at app//org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at app//javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at app//org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at app//org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at app//com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at app//org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at app//org.eclipse.jetty.server.Server.handle(Server.java:563)
        at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at app//org.eclipse.jetty.server.HttpChannel$$Lambda$1319/0x000000014819ac40.dispatch(Unknown Source)
        at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at app//org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at app//org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at app//org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at app//org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at app//org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.runTask(AdaptiveExecutionStrategy.java:416)
        at app//org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.consumeTask(AdaptiveExecutionStrategy.java:385)
        at app//org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:272)
        at app//org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.lambda$new$0(AdaptiveExecutionStrategy.java:140)
        at app//org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy$$Lambda$968/0x0000000146e9f840.run(Unknown Source)
        at app//org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:411)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base@11.0.15/java.lang.Thread.run(Thread.java:829)
   2) Thread[id=58, name=qtp1835695775-58, state=RUNNABLE, group=TGRP-S3BackupRepositoryTest]
        at java.base@11.0.15/sun.nio.ch.FileDispatcherImpl.read0(Native Method)
        at java.base@11.0.15/sun.nio.ch.FileDispatcherImpl.read(FileDispatcherImpl.java:48)
        at java.base@11.0.15/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:276)
        at java.base@11.0.15/sun.nio.ch.IOUtil.read(IOUtil.java:245)
        at java.base@11.0.15/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:223)
        at java.base@11.0.15/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:65)
        at java.base@11.0.15/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:107)
        at java.base@11.0.15/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:101)
        at java.base@11.0.15/java.io.InputStream.read(InputStream.java:205)
        at java.base@11.0.15/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:81)
        at app//com.adobe.testing.s3mock.util.AwsChunkedDecodingInputStream.read(AwsChunkedDecodingInputStream.java:91)
        at java.base@11.0.15/java.io.InputStream.read(InputStream.java:280)
        at java.base@11.0.15/java.io.InputStream.read(InputStream.java:205)
        at app//com.adobe.testing.s3mock.store.ObjectStore.inputStreamToFile(ObjectStore.java:370)
        at app//com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:135)
        at app//com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at app//com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at jdk.internal.reflect.GeneratedMethodAccessor120.invoke(Unknown Source)
        at java.base@11.0.15/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.base@11.0.15/java.lang.reflect.Method.invoke(Method.java:566)
        at app//org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at app//org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at app//org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at app//org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at app//org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at app//org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at app//org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at app//org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at app//org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at app//org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at app//javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at app//org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at app//javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at app//org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at app//org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at app//com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at app//org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at app//org.eclipse.jetty.server.Server.handle(Server.java:563)
        at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at app//org.eclipse.jetty.server.HttpChannel$$Lambda$1319/0x000000014819ac40.dispatch(Unknown Source)
        at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at app//org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at app//org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at app//org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at app//org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base@11.0.15/java.lang.Thread.run(Thread.java:829)
	at __randomizedtesting.SeedInfo.seed([BEB894D975CD3FFD]:0)

[JENKINS] Solr-main-MacOSX (64bit/hotspot/jdk-16.0.2) - Build # 2603 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/2603/
Java: 64bit/hotspot/jdk-16.0.2 -XX:-UseCompressedOops -XX:+UseG1GC

3 tests failed.
FAILED:  org.apache.solr.s3.S3BackupRepositoryTest.testCopyFiles

Error Message:
software.amazon.awssdk.core.exception.SdkClientException: Unable to execute HTTP request: Read timed out

Stack Trace:
software.amazon.awssdk.core.exception.SdkClientException: Unable to execute HTTP request: Read timed out
	at __randomizedtesting.SeedInfo.seed([CC871781036D213E:25B23B529D30C4E]:0)
	at app//software.amazon.awssdk.core.exception.SdkClientException$BuilderImpl.build(SdkClientException.java:111)
	at app//software.amazon.awssdk.core.exception.SdkClientException.create(SdkClientException.java:47)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.utils.RetryableStageHelper.setLastException(RetryableStageHelper.java:223)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.RetryableStage.execute(RetryableStage.java:83)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.RetryableStage.execute(RetryableStage.java:36)
	at app//software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206)
	at app//software.amazon.awssdk.core.internal.http.StreamManagingStage.execute(StreamManagingStage.java:56)
	at app//software.amazon.awssdk.core.internal.http.StreamManagingStage.execute(StreamManagingStage.java:36)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallTimeoutTrackingStage.executeWithTimer(ApiCallTimeoutTrackingStage.java:80)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallTimeoutTrackingStage.execute(ApiCallTimeoutTrackingStage.java:60)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallTimeoutTrackingStage.execute(ApiCallTimeoutTrackingStage.java:42)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallMetricCollectionStage.execute(ApiCallMetricCollectionStage.java:48)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallMetricCollectionStage.execute(ApiCallMetricCollectionStage.java:31)
	at app//software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206)
	at app//software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.ExecutionFailureExceptionReportingStage.execute(ExecutionFailureExceptionReportingStage.java:37)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.ExecutionFailureExceptionReportingStage.execute(ExecutionFailureExceptionReportingStage.java:26)
	at app//software.amazon.awssdk.core.internal.http.AmazonSyncHttpClient$RequestExecutionBuilderImpl.execute(AmazonSyncHttpClient.java:193)
	at app//software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.invoke(BaseSyncClientHandler.java:103)
	at app//software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.doExecute(BaseSyncClientHandler.java:171)
	at app//software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.lambda$execute$1(BaseSyncClientHandler.java:82)
	at app//software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.measureApiCallSuccess(BaseSyncClientHandler.java:179)
	at app//software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.execute(BaseSyncClientHandler.java:76)
	at app//software.amazon.awssdk.core.client.handler.SdkSyncClientHandler.execute(SdkSyncClientHandler.java:45)
	at app//software.amazon.awssdk.awscore.client.handler.AwsSyncClientHandler.execute(AwsSyncClientHandler.java:56)
	at app//software.amazon.awssdk.services.s3.DefaultS3Client.putObject(DefaultS3Client.java:9326)
	at app//software.amazon.awssdk.services.s3.S3Client.putObject(S3Client.java:17280)
	at app//org.apache.solr.s3.S3BackupRepositoryTest.pushObject(S3BackupRepositoryTest.java:328)
	at app//org.apache.solr.s3.S3BackupRepositoryTest.doTestCopyFileTo(S3BackupRepositoryTest.java:214)
	at app//org.apache.solr.s3.S3BackupRepositoryTest.testCopyFiles(S3BackupRepositoryTest.java:173)
	at java.base@16.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base@16.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:78)
	at java.base@16.0.2/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base@16.0.2/java.lang.reflect.Method.invoke(Method.java:567)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at app//org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:54)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at app//com.adobe.testing.s3mock.junit4.S3MockRule$1.evaluate(S3MockRule.java:66)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
	at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
	Suppressed: software.amazon.awssdk.core.exception.SdkClientException: Request attempt 1 failure: Unable to execute HTTP request: Read timed out
	Suppressed: software.amazon.awssdk.core.exception.SdkClientException: Request attempt 2 failure: Unable to execute HTTP request: Read timed out
	Suppressed: software.amazon.awssdk.core.exception.SdkClientException: Request attempt 3 failure: Unable to execute HTTP request: Read timed out
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.base/sun.nio.ch.NioSocketImpl.timedRead(NioSocketImpl.java:283)
	at java.base/sun.nio.ch.NioSocketImpl.implRead(NioSocketImpl.java:309)
	at java.base/sun.nio.ch.NioSocketImpl.read(NioSocketImpl.java:350)
	at java.base/sun.nio.ch.NioSocketImpl$1.read(NioSocketImpl.java:803)
	at java.base/java.net.Socket$SocketInputStream.read(Socket.java:976)
	at java.base/java.io.BufferedInputStream.fill(BufferedInputStream.java:244)
	at java.base/java.io.BufferedInputStream.read1(BufferedInputStream.java:284)
	at java.base/java.io.BufferedInputStream.read(BufferedInputStream.java:343)
	at java.base/sun.net.www.http.HttpClient.parseHTTPHeader(HttpClient.java:788)
	at java.base/sun.net.www.http.HttpClient.parseHTTP(HttpClient.java:723)
	at java.base/sun.net.www.protocol.http.HttpURLConnection.getInputStream0(HttpURLConnection.java:1676)
	at java.base/sun.net.www.protocol.http.HttpURLConnection.getInputStream(HttpURLConnection.java:1577)
	at java.base/java.net.HttpURLConnection.getResponseCode(HttpURLConnection.java:527)
	at software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient$RequestCallable.getResponseCodeSafely(UrlConnectionHttpClient.java:419)
	at software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient$RequestCallable.call(UrlConnectionHttpClient.java:302)
	at software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient$RequestCallable.call(UrlConnectionHttpClient.java:269)
	at software.amazon.awssdk.core.internal.util.MetricUtils.measureDurationUnsafe(MetricUtils.java:63)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.MakeHttpRequestStage.executeHttpRequest(MakeHttpRequestStage.java:77)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.MakeHttpRequestStage.execute(MakeHttpRequestStage.java:56)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.MakeHttpRequestStage.execute(MakeHttpRequestStage.java:39)
	at software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206)
	at software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206)
	at software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206)
	at software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallAttemptTimeoutTrackingStage.execute(ApiCallAttemptTimeoutTrackingStage.java:73)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallAttemptTimeoutTrackingStage.execute(ApiCallAttemptTimeoutTrackingStage.java:42)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.TimeoutExceptionHandlingStage.execute(TimeoutExceptionHandlingStage.java:78)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.TimeoutExceptionHandlingStage.execute(TimeoutExceptionHandlingStage.java:40)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallAttemptMetricCollectionStage.execute(ApiCallAttemptMetricCollectionStage.java:50)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallAttemptMetricCollectionStage.execute(ApiCallAttemptMetricCollectionStage.java:36)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.RetryableStage.execute(RetryableStage.java:81)
	... 75 more


FAILED:  org.apache.solr.s3.S3BackupRepositoryTest.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.s3.S3BackupRepositoryTest: 
   1) Thread[id=61, name=qtp24566309-61, state=BLOCKED, group=TGRP-S3BackupRepositoryTest]
        at app//com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:133)
        at app//com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at app//com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at jdk.internal.reflect.GeneratedMethodAccessor120.invoke(Unknown Source)
        at java.base@16.0.2/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.base@16.0.2/java.lang.reflect.Method.invoke(Method.java:567)
        at app//org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at app//org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at app//org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at app//org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at app//org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at app//org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at app//org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at app//org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at app//org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at app//org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at app//javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at app//org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at app//javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at app//org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at app//org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at app//com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at app//org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at app//org.eclipse.jetty.server.Server.handle(Server.java:563)
        at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at app//org.eclipse.jetty.server.HttpChannel$$Lambda$1387/0x00000008013a0000.dispatch(Unknown Source)
        at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at app//org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at app//org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at app//org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at app//org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   2) Thread[id=57, name=qtp24566309-57, state=RUNNABLE, group=TGRP-S3BackupRepositoryTest]
        at java.base@16.0.2/sun.nio.ch.FileDispatcherImpl.read0(Native Method)
        at java.base@16.0.2/sun.nio.ch.FileDispatcherImpl.read(FileDispatcherImpl.java:48)
        at java.base@16.0.2/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:276)
        at java.base@16.0.2/sun.nio.ch.IOUtil.read(IOUtil.java:245)
        at java.base@16.0.2/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:229)
        at java.base@16.0.2/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:65)
        at java.base@16.0.2/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:107)
        at java.base@16.0.2/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:101)
        at java.base@16.0.2/java.io.InputStream.read(InputStream.java:218)
        at java.base@16.0.2/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:81)
        at app//com.adobe.testing.s3mock.util.AwsChunkedDecodingInputStream.read(AwsChunkedDecodingInputStream.java:91)
        at java.base@16.0.2/java.io.InputStream.read(InputStream.java:293)
        at java.base@16.0.2/java.io.InputStream.read(InputStream.java:218)
        at app//com.adobe.testing.s3mock.store.ObjectStore.inputStreamToFile(ObjectStore.java:370)
        at app//com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:135)
        at app//com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at app//com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at java.base@16.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at java.base@16.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:78)
        at java.base@16.0.2/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.base@16.0.2/java.lang.reflect.Method.invoke(Method.java:567)
        at app//org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at app//org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at app//org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at app//org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at app//org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at app//org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at app//org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at app//org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at app//org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at app//org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at app//javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at app//org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at app//javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at app//org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at app//org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at app//com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at app//org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at app//org.eclipse.jetty.server.Server.handle(Server.java:563)
        at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at app//org.eclipse.jetty.server.HttpChannel$$Lambda$1387/0x00000008013a0000.dispatch(Unknown Source)
        at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at app//org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at app//org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at app//org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at app//org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.s3.S3BackupRepositoryTest: 
   1) Thread[id=61, name=qtp24566309-61, state=BLOCKED, group=TGRP-S3BackupRepositoryTest]
        at app//com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:133)
        at app//com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at app//com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at jdk.internal.reflect.GeneratedMethodAccessor120.invoke(Unknown Source)
        at java.base@16.0.2/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.base@16.0.2/java.lang.reflect.Method.invoke(Method.java:567)
        at app//org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at app//org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at app//org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at app//org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at app//org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at app//org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at app//org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at app//org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at app//org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at app//org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at app//javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at app//org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at app//javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at app//org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at app//org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at app//com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at app//org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at app//org.eclipse.jetty.server.Server.handle(Server.java:563)
        at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at app//org.eclipse.jetty.server.HttpChannel$$Lambda$1387/0x00000008013a0000.dispatch(Unknown Source)
        at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at app//org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at app//org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at app//org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at app//org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   2) Thread[id=57, name=qtp24566309-57, state=RUNNABLE, group=TGRP-S3BackupRepositoryTest]
        at java.base@16.0.2/sun.nio.ch.FileDispatcherImpl.read0(Native Method)
        at java.base@16.0.2/sun.nio.ch.FileDispatcherImpl.read(FileDispatcherImpl.java:48)
        at java.base@16.0.2/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:276)
        at java.base@16.0.2/sun.nio.ch.IOUtil.read(IOUtil.java:245)
        at java.base@16.0.2/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:229)
        at java.base@16.0.2/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:65)
        at java.base@16.0.2/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:107)
        at java.base@16.0.2/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:101)
        at java.base@16.0.2/java.io.InputStream.read(InputStream.java:218)
        at java.base@16.0.2/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:81)
        at app//com.adobe.testing.s3mock.util.AwsChunkedDecodingInputStream.read(AwsChunkedDecodingInputStream.java:91)
        at java.base@16.0.2/java.io.InputStream.read(InputStream.java:293)
        at java.base@16.0.2/java.io.InputStream.read(InputStream.java:218)
        at app//com.adobe.testing.s3mock.store.ObjectStore.inputStreamToFile(ObjectStore.java:370)
        at app//com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:135)
        at app//com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at app//com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at java.base@16.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at java.base@16.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:78)
        at java.base@16.0.2/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.base@16.0.2/java.lang.reflect.Method.invoke(Method.java:567)
        at app//org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at app//org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at app//org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at app//org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at app//org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at app//org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at app//org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at app//org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at app//org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at app//org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at app//javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at app//org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at app//javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at app//org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at app//org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at app//com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at app//org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at app//org.eclipse.jetty.server.Server.handle(Server.java:563)
        at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at app//org.eclipse.jetty.server.HttpChannel$$Lambda$1387/0x00000008013a0000.dispatch(Unknown Source)
        at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at app//org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at app//org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at app//org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at app//org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
	at __randomizedtesting.SeedInfo.seed([CC871781036D213E]:0)


FAILED:  org.apache.solr.s3.S3BackupRepositoryTest.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=61, name=qtp24566309-61, state=BLOCKED, group=TGRP-S3BackupRepositoryTest]
        at app//com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:133)
        at app//com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at app//com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at jdk.internal.reflect.GeneratedMethodAccessor120.invoke(Unknown Source)
        at java.base@16.0.2/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.base@16.0.2/java.lang.reflect.Method.invoke(Method.java:567)
        at app//org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at app//org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at app//org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at app//org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at app//org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at app//org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at app//org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at app//org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at app//org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at app//org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at app//javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at app//org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at app//javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at app//org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at app//org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at app//com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at app//org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at app//org.eclipse.jetty.server.Server.handle(Server.java:563)
        at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at app//org.eclipse.jetty.server.HttpChannel$$Lambda$1387/0x00000008013a0000.dispatch(Unknown Source)
        at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at app//org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at app//org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at app//org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at app//org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   2) Thread[id=57, name=qtp24566309-57, state=RUNNABLE, group=TGRP-S3BackupRepositoryTest]
        at java.base@16.0.2/sun.nio.ch.FileDispatcherImpl.read0(Native Method)
        at java.base@16.0.2/sun.nio.ch.FileDispatcherImpl.read(FileDispatcherImpl.java:48)
        at java.base@16.0.2/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:276)
        at java.base@16.0.2/sun.nio.ch.IOUtil.read(IOUtil.java:245)
        at java.base@16.0.2/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:229)
        at java.base@16.0.2/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:65)
        at java.base@16.0.2/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:107)
        at java.base@16.0.2/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:101)
        at java.base@16.0.2/java.io.InputStream.read(InputStream.java:218)
        at java.base@16.0.2/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:81)
        at app//com.adobe.testing.s3mock.util.AwsChunkedDecodingInputStream.read(AwsChunkedDecodingInputStream.java:91)
        at java.base@16.0.2/java.io.InputStream.read(InputStream.java:293)
        at java.base@16.0.2/java.io.InputStream.read(InputStream.java:218)
        at app//com.adobe.testing.s3mock.store.ObjectStore.inputStreamToFile(ObjectStore.java:370)
        at app//com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:135)
        at app//com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at app//com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at java.base@16.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at java.base@16.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:78)
        at java.base@16.0.2/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.base@16.0.2/java.lang.reflect.Method.invoke(Method.java:567)
        at app//org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at app//org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at app//org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at app//org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at app//org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at app//org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at app//org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at app//org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at app//org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at app//org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at app//javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at app//org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at app//javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at app//org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at app//org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at app//com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at app//org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at app//org.eclipse.jetty.server.Server.handle(Server.java:563)
        at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at app//org.eclipse.jetty.server.HttpChannel$$Lambda$1387/0x00000008013a0000.dispatch(Unknown Source)
        at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at app//org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at app//org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at app//org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at app//org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=61, name=qtp24566309-61, state=BLOCKED, group=TGRP-S3BackupRepositoryTest]
        at app//com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:133)
        at app//com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at app//com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at jdk.internal.reflect.GeneratedMethodAccessor120.invoke(Unknown Source)
        at java.base@16.0.2/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.base@16.0.2/java.lang.reflect.Method.invoke(Method.java:567)
        at app//org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at app//org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at app//org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at app//org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at app//org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at app//org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at app//org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at app//org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at app//org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at app//org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at app//javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at app//org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at app//javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at app//org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at app//org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at app//com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at app//org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at app//org.eclipse.jetty.server.Server.handle(Server.java:563)
        at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at app//org.eclipse.jetty.server.HttpChannel$$Lambda$1387/0x00000008013a0000.dispatch(Unknown Source)
        at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at app//org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at app//org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at app//org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at app//org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   2) Thread[id=57, name=qtp24566309-57, state=RUNNABLE, group=TGRP-S3BackupRepositoryTest]
        at java.base@16.0.2/sun.nio.ch.FileDispatcherImpl.read0(Native Method)
        at java.base@16.0.2/sun.nio.ch.FileDispatcherImpl.read(FileDispatcherImpl.java:48)
        at java.base@16.0.2/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:276)
        at java.base@16.0.2/sun.nio.ch.IOUtil.read(IOUtil.java:245)
        at java.base@16.0.2/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:229)
        at java.base@16.0.2/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:65)
        at java.base@16.0.2/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:107)
        at java.base@16.0.2/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:101)
        at java.base@16.0.2/java.io.InputStream.read(InputStream.java:218)
        at java.base@16.0.2/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:81)
        at app//com.adobe.testing.s3mock.util.AwsChunkedDecodingInputStream.read(AwsChunkedDecodingInputStream.java:91)
        at java.base@16.0.2/java.io.InputStream.read(InputStream.java:293)
        at java.base@16.0.2/java.io.InputStream.read(InputStream.java:218)
        at app//com.adobe.testing.s3mock.store.ObjectStore.inputStreamToFile(ObjectStore.java:370)
        at app//com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:135)
        at app//com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at app//com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at java.base@16.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at java.base@16.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:78)
        at java.base@16.0.2/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.base@16.0.2/java.lang.reflect.Method.invoke(Method.java:567)
        at app//org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at app//org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at app//org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at app//org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at app//org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at app//org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at app//org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at app//org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at app//org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at app//org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at app//javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at app//org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at app//javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at app//org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at app//org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at app//com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at app//org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at app//org.eclipse.jetty.server.Server.handle(Server.java:563)
        at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at app//org.eclipse.jetty.server.HttpChannel$$Lambda$1387/0x00000008013a0000.dispatch(Unknown Source)
        at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at app//org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at app//org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at app//org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at app//org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
	at __randomizedtesting.SeedInfo.seed([CC871781036D213E]:0)

[JENKINS] Solr-main-MacOSX (64bit/hotspot/jdk-16.0.2) - Build # 2602 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/2602/
Java: 64bit/hotspot/jdk-16.0.2 -XX:-UseCompressedOops -XX:+UseG1GC

3 tests failed.
FAILED:  org.apache.solr.s3.S3BackupRepositoryTest.testCopyFiles

Error Message:
software.amazon.awssdk.core.exception.SdkClientException: Unable to execute HTTP request: Read timed out

Stack Trace:
software.amazon.awssdk.core.exception.SdkClientException: Unable to execute HTTP request: Read timed out
	at __randomizedtesting.SeedInfo.seed([9AF24E14CFBDCFB8:542E7A20E503E2C8]:0)
	at app//software.amazon.awssdk.core.exception.SdkClientException$BuilderImpl.build(SdkClientException.java:111)
	at app//software.amazon.awssdk.core.exception.SdkClientException.create(SdkClientException.java:47)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.utils.RetryableStageHelper.setLastException(RetryableStageHelper.java:223)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.RetryableStage.execute(RetryableStage.java:83)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.RetryableStage.execute(RetryableStage.java:36)
	at app//software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206)
	at app//software.amazon.awssdk.core.internal.http.StreamManagingStage.execute(StreamManagingStage.java:56)
	at app//software.amazon.awssdk.core.internal.http.StreamManagingStage.execute(StreamManagingStage.java:36)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallTimeoutTrackingStage.executeWithTimer(ApiCallTimeoutTrackingStage.java:80)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallTimeoutTrackingStage.execute(ApiCallTimeoutTrackingStage.java:60)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallTimeoutTrackingStage.execute(ApiCallTimeoutTrackingStage.java:42)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallMetricCollectionStage.execute(ApiCallMetricCollectionStage.java:48)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallMetricCollectionStage.execute(ApiCallMetricCollectionStage.java:31)
	at app//software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206)
	at app//software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.ExecutionFailureExceptionReportingStage.execute(ExecutionFailureExceptionReportingStage.java:37)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.ExecutionFailureExceptionReportingStage.execute(ExecutionFailureExceptionReportingStage.java:26)
	at app//software.amazon.awssdk.core.internal.http.AmazonSyncHttpClient$RequestExecutionBuilderImpl.execute(AmazonSyncHttpClient.java:193)
	at app//software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.invoke(BaseSyncClientHandler.java:103)
	at app//software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.doExecute(BaseSyncClientHandler.java:171)
	at app//software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.lambda$execute$1(BaseSyncClientHandler.java:82)
	at app//software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.measureApiCallSuccess(BaseSyncClientHandler.java:179)
	at app//software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.execute(BaseSyncClientHandler.java:76)
	at app//software.amazon.awssdk.core.client.handler.SdkSyncClientHandler.execute(SdkSyncClientHandler.java:45)
	at app//software.amazon.awssdk.awscore.client.handler.AwsSyncClientHandler.execute(AwsSyncClientHandler.java:56)
	at app//software.amazon.awssdk.services.s3.DefaultS3Client.putObject(DefaultS3Client.java:9326)
	at app//software.amazon.awssdk.services.s3.S3Client.putObject(S3Client.java:17280)
	at app//org.apache.solr.s3.S3BackupRepositoryTest.pushObject(S3BackupRepositoryTest.java:328)
	at app//org.apache.solr.s3.S3BackupRepositoryTest.doTestCopyFileTo(S3BackupRepositoryTest.java:214)
	at app//org.apache.solr.s3.S3BackupRepositoryTest.testCopyFiles(S3BackupRepositoryTest.java:173)
	at java.base@16.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base@16.0.2/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:78)
	at java.base@16.0.2/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base@16.0.2/java.lang.reflect.Method.invoke(Method.java:567)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at app//org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:54)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at app//com.adobe.testing.s3mock.junit4.S3MockRule$1.evaluate(S3MockRule.java:66)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
	at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
	Suppressed: software.amazon.awssdk.core.exception.SdkClientException: Request attempt 1 failure: Unable to execute HTTP request: Read timed out
	Suppressed: software.amazon.awssdk.core.exception.SdkClientException: Request attempt 2 failure: Unable to execute HTTP request: Read timed out
	Suppressed: software.amazon.awssdk.core.exception.SdkClientException: Request attempt 3 failure: Unable to execute HTTP request: Read timed out
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.base/sun.nio.ch.NioSocketImpl.timedRead(NioSocketImpl.java:283)
	at java.base/sun.nio.ch.NioSocketImpl.implRead(NioSocketImpl.java:309)
	at java.base/sun.nio.ch.NioSocketImpl.read(NioSocketImpl.java:350)
	at java.base/sun.nio.ch.NioSocketImpl$1.read(NioSocketImpl.java:803)
	at java.base/java.net.Socket$SocketInputStream.read(Socket.java:976)
	at java.base/java.io.BufferedInputStream.fill(BufferedInputStream.java:244)
	at java.base/java.io.BufferedInputStream.read1(BufferedInputStream.java:284)
	at java.base/java.io.BufferedInputStream.read(BufferedInputStream.java:343)
	at java.base/sun.net.www.http.HttpClient.parseHTTPHeader(HttpClient.java:788)
	at java.base/sun.net.www.http.HttpClient.parseHTTP(HttpClient.java:723)
	at java.base/sun.net.www.protocol.http.HttpURLConnection.getInputStream0(HttpURLConnection.java:1676)
	at java.base/sun.net.www.protocol.http.HttpURLConnection.getInputStream(HttpURLConnection.java:1577)
	at java.base/java.net.HttpURLConnection.getResponseCode(HttpURLConnection.java:527)
	at software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient$RequestCallable.getResponseCodeSafely(UrlConnectionHttpClient.java:419)
	at software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient$RequestCallable.call(UrlConnectionHttpClient.java:302)
	at software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient$RequestCallable.call(UrlConnectionHttpClient.java:269)
	at software.amazon.awssdk.core.internal.util.MetricUtils.measureDurationUnsafe(MetricUtils.java:63)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.MakeHttpRequestStage.executeHttpRequest(MakeHttpRequestStage.java:77)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.MakeHttpRequestStage.execute(MakeHttpRequestStage.java:56)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.MakeHttpRequestStage.execute(MakeHttpRequestStage.java:39)
	at software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206)
	at software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206)
	at software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206)
	at software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallAttemptTimeoutTrackingStage.execute(ApiCallAttemptTimeoutTrackingStage.java:73)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallAttemptTimeoutTrackingStage.execute(ApiCallAttemptTimeoutTrackingStage.java:42)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.TimeoutExceptionHandlingStage.execute(TimeoutExceptionHandlingStage.java:78)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.TimeoutExceptionHandlingStage.execute(TimeoutExceptionHandlingStage.java:40)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallAttemptMetricCollectionStage.execute(ApiCallAttemptMetricCollectionStage.java:50)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallAttemptMetricCollectionStage.execute(ApiCallAttemptMetricCollectionStage.java:36)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.RetryableStage.execute(RetryableStage.java:81)
	... 75 more


FAILED:  org.apache.solr.s3.S3BackupRepositoryTest.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.s3.S3BackupRepositoryTest: 
   1) Thread[id=59, name=qtp164885539-59, state=RUNNABLE, group=TGRP-S3BackupRepositoryTest]
        at java.base@16.0.2/sun.nio.ch.FileDispatcherImpl.read0(Native Method)
        at java.base@16.0.2/sun.nio.ch.FileDispatcherImpl.read(FileDispatcherImpl.java:48)
        at java.base@16.0.2/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:276)
        at java.base@16.0.2/sun.nio.ch.IOUtil.read(IOUtil.java:245)
        at java.base@16.0.2/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:229)
        at java.base@16.0.2/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:65)
        at java.base@16.0.2/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:107)
        at java.base@16.0.2/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:101)
        at java.base@16.0.2/java.io.InputStream.read(InputStream.java:218)
        at java.base@16.0.2/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:81)
        at app//com.adobe.testing.s3mock.util.AwsChunkedDecodingInputStream.read(AwsChunkedDecodingInputStream.java:91)
        at java.base@16.0.2/java.io.InputStream.read(InputStream.java:293)
        at java.base@16.0.2/java.io.InputStream.read(InputStream.java:218)
        at app//com.adobe.testing.s3mock.store.ObjectStore.inputStreamToFile(ObjectStore.java:370)
        at app//com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:135)
        at app//com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at app//com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at jdk.internal.reflect.GeneratedMethodAccessor120.invoke(Unknown Source)
        at java.base@16.0.2/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.base@16.0.2/java.lang.reflect.Method.invoke(Method.java:567)
        at app//org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at app//org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at app//org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at app//org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at app//org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at app//org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at app//org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at app//org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at app//org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at app//org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at app//javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at app//org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at app//javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at app//org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at app//org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at app//com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at app//org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at app//org.eclipse.jetty.server.Server.handle(Server.java:563)
        at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at app//org.eclipse.jetty.server.HttpChannel$$Lambda$1388/0x00000008013b5140.dispatch(Unknown Source)
        at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at app//org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at app//org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at app//org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at app//org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   2) Thread[id=63, name=qtp164885539-63, state=BLOCKED, group=TGRP-S3BackupRepositoryTest]
        at app//com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:133)
        at app//com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at app//com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at jdk.internal.reflect.GeneratedMethodAccessor120.invoke(Unknown Source)
        at java.base@16.0.2/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.base@16.0.2/java.lang.reflect.Method.invoke(Method.java:567)
        at app//org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at app//org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at app//org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at app//org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at app//org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at app//org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at app//org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at app//org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at app//org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at app//org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at app//javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at app//org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at app//javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at app//org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at app//org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at app//com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at app//org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at app//org.eclipse.jetty.server.Server.handle(Server.java:563)
        at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at app//org.eclipse.jetty.server.HttpChannel$$Lambda$1388/0x00000008013b5140.dispatch(Unknown Source)
        at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at app//org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at app//org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at app//org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at app//org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.s3.S3BackupRepositoryTest: 
   1) Thread[id=59, name=qtp164885539-59, state=RUNNABLE, group=TGRP-S3BackupRepositoryTest]
        at java.base@16.0.2/sun.nio.ch.FileDispatcherImpl.read0(Native Method)
        at java.base@16.0.2/sun.nio.ch.FileDispatcherImpl.read(FileDispatcherImpl.java:48)
        at java.base@16.0.2/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:276)
        at java.base@16.0.2/sun.nio.ch.IOUtil.read(IOUtil.java:245)
        at java.base@16.0.2/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:229)
        at java.base@16.0.2/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:65)
        at java.base@16.0.2/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:107)
        at java.base@16.0.2/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:101)
        at java.base@16.0.2/java.io.InputStream.read(InputStream.java:218)
        at java.base@16.0.2/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:81)
        at app//com.adobe.testing.s3mock.util.AwsChunkedDecodingInputStream.read(AwsChunkedDecodingInputStream.java:91)
        at java.base@16.0.2/java.io.InputStream.read(InputStream.java:293)
        at java.base@16.0.2/java.io.InputStream.read(InputStream.java:218)
        at app//com.adobe.testing.s3mock.store.ObjectStore.inputStreamToFile(ObjectStore.java:370)
        at app//com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:135)
        at app//com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at app//com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at jdk.internal.reflect.GeneratedMethodAccessor120.invoke(Unknown Source)
        at java.base@16.0.2/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.base@16.0.2/java.lang.reflect.Method.invoke(Method.java:567)
        at app//org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at app//org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at app//org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at app//org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at app//org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at app//org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at app//org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at app//org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at app//org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at app//org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at app//javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at app//org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at app//javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at app//org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at app//org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at app//com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at app//org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at app//org.eclipse.jetty.server.Server.handle(Server.java:563)
        at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at app//org.eclipse.jetty.server.HttpChannel$$Lambda$1388/0x00000008013b5140.dispatch(Unknown Source)
        at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at app//org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at app//org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at app//org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at app//org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   2) Thread[id=63, name=qtp164885539-63, state=BLOCKED, group=TGRP-S3BackupRepositoryTest]
        at app//com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:133)
        at app//com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at app//com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at jdk.internal.reflect.GeneratedMethodAccessor120.invoke(Unknown Source)
        at java.base@16.0.2/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.base@16.0.2/java.lang.reflect.Method.invoke(Method.java:567)
        at app//org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at app//org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at app//org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at app//org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at app//org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at app//org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at app//org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at app//org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at app//org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at app//org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at app//javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at app//org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at app//javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at app//org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at app//org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at app//com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at app//org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at app//org.eclipse.jetty.server.Server.handle(Server.java:563)
        at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at app//org.eclipse.jetty.server.HttpChannel$$Lambda$1388/0x00000008013b5140.dispatch(Unknown Source)
        at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at app//org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at app//org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at app//org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at app//org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
	at __randomizedtesting.SeedInfo.seed([9AF24E14CFBDCFB8]:0)


FAILED:  org.apache.solr.s3.S3BackupRepositoryTest.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=59, name=qtp164885539-59, state=RUNNABLE, group=TGRP-S3BackupRepositoryTest]
        at java.base@16.0.2/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:229)
        at java.base@16.0.2/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:65)
        at java.base@16.0.2/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:107)
        at java.base@16.0.2/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:101)
        at java.base@16.0.2/java.io.InputStream.read(InputStream.java:218)
        at java.base@16.0.2/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:81)
        at app//com.adobe.testing.s3mock.util.AwsChunkedDecodingInputStream.read(AwsChunkedDecodingInputStream.java:91)
        at java.base@16.0.2/java.io.InputStream.read(InputStream.java:293)
        at java.base@16.0.2/java.io.InputStream.read(InputStream.java:218)
        at app//com.adobe.testing.s3mock.store.ObjectStore.inputStreamToFile(ObjectStore.java:370)
        at app//com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:135)
        at app//com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at app//com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at jdk.internal.reflect.GeneratedMethodAccessor120.invoke(Unknown Source)
        at java.base@16.0.2/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.base@16.0.2/java.lang.reflect.Method.invoke(Method.java:567)
        at app//org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at app//org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at app//org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at app//org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at app//org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at app//org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at app//org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at app//org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at app//org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at app//org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at app//javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at app//org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at app//javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at app//org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at app//org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at app//com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at app//org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at app//org.eclipse.jetty.server.Server.handle(Server.java:563)
        at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at app//org.eclipse.jetty.server.HttpChannel$$Lambda$1388/0x00000008013b5140.dispatch(Unknown Source)
        at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at app//org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at app//org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at app//org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at app//org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   2) Thread[id=63, name=qtp164885539-63, state=BLOCKED, group=TGRP-S3BackupRepositoryTest]
        at app//com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:133)
        at app//com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at app//com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at jdk.internal.reflect.GeneratedMethodAccessor120.invoke(Unknown Source)
        at java.base@16.0.2/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.base@16.0.2/java.lang.reflect.Method.invoke(Method.java:567)
        at app//org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at app//org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at app//org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at app//org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at app//org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at app//org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at app//org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at app//org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at app//org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at app//org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at app//javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at app//org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at app//javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at app//org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at app//org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at app//com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at app//org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at app//org.eclipse.jetty.server.Server.handle(Server.java:563)
        at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at app//org.eclipse.jetty.server.HttpChannel$$Lambda$1388/0x00000008013b5140.dispatch(Unknown Source)
        at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at app//org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at app//org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at app//org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at app//org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=59, name=qtp164885539-59, state=RUNNABLE, group=TGRP-S3BackupRepositoryTest]
        at java.base@16.0.2/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:229)
        at java.base@16.0.2/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:65)
        at java.base@16.0.2/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:107)
        at java.base@16.0.2/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:101)
        at java.base@16.0.2/java.io.InputStream.read(InputStream.java:218)
        at java.base@16.0.2/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:81)
        at app//com.adobe.testing.s3mock.util.AwsChunkedDecodingInputStream.read(AwsChunkedDecodingInputStream.java:91)
        at java.base@16.0.2/java.io.InputStream.read(InputStream.java:293)
        at java.base@16.0.2/java.io.InputStream.read(InputStream.java:218)
        at app//com.adobe.testing.s3mock.store.ObjectStore.inputStreamToFile(ObjectStore.java:370)
        at app//com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:135)
        at app//com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at app//com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at jdk.internal.reflect.GeneratedMethodAccessor120.invoke(Unknown Source)
        at java.base@16.0.2/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.base@16.0.2/java.lang.reflect.Method.invoke(Method.java:567)
        at app//org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at app//org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at app//org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at app//org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at app//org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at app//org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at app//org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at app//org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at app//org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at app//org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at app//javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at app//org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at app//javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at app//org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at app//org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at app//com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at app//org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at app//org.eclipse.jetty.server.Server.handle(Server.java:563)
        at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at app//org.eclipse.jetty.server.HttpChannel$$Lambda$1388/0x00000008013b5140.dispatch(Unknown Source)
        at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at app//org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at app//org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at app//org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at app//org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
   2) Thread[id=63, name=qtp164885539-63, state=BLOCKED, group=TGRP-S3BackupRepositoryTest]
        at app//com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:133)
        at app//com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at app//com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at jdk.internal.reflect.GeneratedMethodAccessor120.invoke(Unknown Source)
        at java.base@16.0.2/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.base@16.0.2/java.lang.reflect.Method.invoke(Method.java:567)
        at app//org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at app//org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at app//org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at app//org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at app//org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at app//org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at app//org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at app//org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at app//org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at app//org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at app//javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at app//org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at app//javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at app//org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at app//org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at app//com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at app//org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at app//org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at app//org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at app//org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at app//org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at app//org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at app//org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at app//org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at app//org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at app//org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at app//org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at app//org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at app//org.eclipse.jetty.server.Server.handle(Server.java:563)
        at app//org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at app//org.eclipse.jetty.server.HttpChannel$$Lambda$1388/0x00000008013b5140.dispatch(Unknown Source)
        at app//org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at app//org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at app//org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at app//org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at app//org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at app//org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
	at __randomizedtesting.SeedInfo.seed([9AF24E14CFBDCFB8]:0)

[JENKINS] Solr-main-MacOSX (64bit/hotspot/jdk-19) - Build # 2601 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/2601/
Java: 64bit/hotspot/jdk-19 -XX:-UseCompressedOops -XX:+UseSerialGC

4 tests failed.
FAILED:  org.apache.solr.cloud.ReindexCollectionTest.testAbort

Error Message:
java.lang.AssertionError: {responseHeader={status=0, QTime=14}, status={state=running, msg=found [b2314bda-3cfb-494c-ab4a-7b97fb1e8282] in running tasks}}

Stack Trace:
java.lang.AssertionError: {responseHeader={status=0, QTime=14}, status={state=running, msg=found [b2314bda-3cfb-494c-ab4a-7b97fb1e8282] in running tasks}}
	at __randomizedtesting.SeedInfo.seed([AF876C82CAA2922F:C9E4DB15BE780056]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.junit.Assert.assertTrue(Assert.java:42)
	at org.junit.Assert.assertNotNull(Assert.java:713)
	at org.apache.solr.cloud.ReindexCollectionTest.testAbort(ReindexCollectionTest.java:425)
	at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
	at java.base/java.lang.reflect.Method.invoke(Method.java:578)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
	at java.base/java.lang.Thread.run(Thread.java:1589)


FAILED:  org.apache.solr.s3.S3BackupRepositoryTest.testCopyFiles

Error Message:
software.amazon.awssdk.core.exception.SdkClientException: Unable to execute HTTP request: Read timed out

Stack Trace:
software.amazon.awssdk.core.exception.SdkClientException: Unable to execute HTTP request: Read timed out
	at __randomizedtesting.SeedInfo.seed([AF876C82CAA2922F:615B58B6E01CBF5F]:0)
	at app//software.amazon.awssdk.core.exception.SdkClientException$BuilderImpl.build(SdkClientException.java:111)
	at app//software.amazon.awssdk.core.exception.SdkClientException.create(SdkClientException.java:47)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.utils.RetryableStageHelper.setLastException(RetryableStageHelper.java:223)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.RetryableStage.execute(RetryableStage.java:83)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.RetryableStage.execute(RetryableStage.java:36)
	at app//software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206)
	at app//software.amazon.awssdk.core.internal.http.StreamManagingStage.execute(StreamManagingStage.java:56)
	at app//software.amazon.awssdk.core.internal.http.StreamManagingStage.execute(StreamManagingStage.java:36)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallTimeoutTrackingStage.executeWithTimer(ApiCallTimeoutTrackingStage.java:80)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallTimeoutTrackingStage.execute(ApiCallTimeoutTrackingStage.java:60)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallTimeoutTrackingStage.execute(ApiCallTimeoutTrackingStage.java:42)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallMetricCollectionStage.execute(ApiCallMetricCollectionStage.java:48)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallMetricCollectionStage.execute(ApiCallMetricCollectionStage.java:31)
	at app//software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206)
	at app//software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.ExecutionFailureExceptionReportingStage.execute(ExecutionFailureExceptionReportingStage.java:37)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.ExecutionFailureExceptionReportingStage.execute(ExecutionFailureExceptionReportingStage.java:26)
	at app//software.amazon.awssdk.core.internal.http.AmazonSyncHttpClient$RequestExecutionBuilderImpl.execute(AmazonSyncHttpClient.java:193)
	at app//software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.invoke(BaseSyncClientHandler.java:103)
	at app//software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.doExecute(BaseSyncClientHandler.java:171)
	at app//software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.lambda$execute$1(BaseSyncClientHandler.java:82)
	at app//software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.measureApiCallSuccess(BaseSyncClientHandler.java:179)
	at app//software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.execute(BaseSyncClientHandler.java:76)
	at app//software.amazon.awssdk.core.client.handler.SdkSyncClientHandler.execute(SdkSyncClientHandler.java:45)
	at app//software.amazon.awssdk.awscore.client.handler.AwsSyncClientHandler.execute(AwsSyncClientHandler.java:56)
	at app//software.amazon.awssdk.services.s3.DefaultS3Client.putObject(DefaultS3Client.java:9326)
	at app//software.amazon.awssdk.services.s3.S3Client.putObject(S3Client.java:17280)
	at app//org.apache.solr.s3.S3BackupRepositoryTest.pushObject(S3BackupRepositoryTest.java:328)
	at app//org.apache.solr.s3.S3BackupRepositoryTest.doTestCopyFileTo(S3BackupRepositoryTest.java:214)
	at app//org.apache.solr.s3.S3BackupRepositoryTest.testCopyFiles(S3BackupRepositoryTest.java:173)
	at java.base@19/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
	at java.base@19/java.lang.reflect.Method.invoke(Method.java:578)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at app//org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:54)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at app//com.adobe.testing.s3mock.junit4.S3MockRule$1.evaluate(S3MockRule.java:66)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
	at java.base@19/java.lang.Thread.run(Thread.java:1589)
	Suppressed: software.amazon.awssdk.core.exception.SdkClientException: Request attempt 1 failure: Unable to execute HTTP request: Read timed out
	Suppressed: software.amazon.awssdk.core.exception.SdkClientException: Request attempt 2 failure: Unable to execute HTTP request: Read timed out
	Suppressed: software.amazon.awssdk.core.exception.SdkClientException: Request attempt 3 failure: Unable to execute HTTP request: Read timed out
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.base/sun.nio.ch.NioSocketImpl.timedRead(NioSocketImpl.java:273)
	at java.base/sun.nio.ch.NioSocketImpl.implRead(NioSocketImpl.java:299)
	at java.base/sun.nio.ch.NioSocketImpl.read(NioSocketImpl.java:340)
	at java.base/sun.nio.ch.NioSocketImpl$1.read(NioSocketImpl.java:789)
	at java.base/java.net.Socket$SocketInputStream.read(Socket.java:1025)
	at java.base/java.io.BufferedInputStream.fill(BufferedInputStream.java:255)
	at java.base/java.io.BufferedInputStream.read1(BufferedInputStream.java:310)
	at java.base/java.io.BufferedInputStream.implRead(BufferedInputStream.java:382)
	at java.base/java.io.BufferedInputStream.read(BufferedInputStream.java:361)
	at java.base/sun.net.www.http.HttpClient.parseHTTPHeader(HttpClient.java:827)
	at java.base/sun.net.www.http.HttpClient.parseHTTP(HttpClient.java:759)
	at java.base/sun.net.www.protocol.http.HttpURLConnection.getInputStream0(HttpURLConnection.java:1684)
	at java.base/sun.net.www.protocol.http.HttpURLConnection.getInputStream(HttpURLConnection.java:1585)
	at java.base/java.net.HttpURLConnection.getResponseCode(HttpURLConnection.java:529)
	at software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient$RequestCallable.getResponseCodeSafely(UrlConnectionHttpClient.java:419)
	at software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient$RequestCallable.call(UrlConnectionHttpClient.java:302)
	at software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient$RequestCallable.call(UrlConnectionHttpClient.java:269)
	at software.amazon.awssdk.core.internal.util.MetricUtils.measureDurationUnsafe(MetricUtils.java:63)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.MakeHttpRequestStage.executeHttpRequest(MakeHttpRequestStage.java:77)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.MakeHttpRequestStage.execute(MakeHttpRequestStage.java:56)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.MakeHttpRequestStage.execute(MakeHttpRequestStage.java:39)
	at software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206)
	at software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206)
	at software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206)
	at software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallAttemptTimeoutTrackingStage.execute(ApiCallAttemptTimeoutTrackingStage.java:73)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallAttemptTimeoutTrackingStage.execute(ApiCallAttemptTimeoutTrackingStage.java:42)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.TimeoutExceptionHandlingStage.execute(TimeoutExceptionHandlingStage.java:78)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.TimeoutExceptionHandlingStage.execute(TimeoutExceptionHandlingStage.java:40)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallAttemptMetricCollectionStage.execute(ApiCallAttemptMetricCollectionStage.java:50)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallAttemptMetricCollectionStage.execute(ApiCallAttemptMetricCollectionStage.java:36)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.RetryableStage.execute(RetryableStage.java:81)
	... 73 more


FAILED:  org.apache.solr.s3.S3BackupRepositoryTest.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 3 threads leaked from SUITE scope at org.apache.solr.s3.S3BackupRepositoryTest: 
   1) Thread[id=58, name=qtp1737344236-58, state=BLOCKED, group=TGRP-S3BackupRepositoryTest]
        at com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:133)
        at com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
        at java.base/java.lang.reflect.Method.invoke(Method.java:578)
        at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.Server.handle(Server.java:563)
        at org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base/java.lang.Thread.run(Thread.java:1589)
   2) Thread[id=66, name=qtp1737344236-66, state=RUNNABLE, group=TGRP-S3BackupRepositoryTest]
        at java.base/sun.nio.ch.FileDispatcherImpl.read0(Native Method)
        at java.base/sun.nio.ch.FileDispatcherImpl.read(FileDispatcherImpl.java:48)
        at java.base/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:339)
        at java.base/sun.nio.ch.IOUtil.read(IOUtil.java:305)
        at java.base/sun.nio.ch.IOUtil.read(IOUtil.java:282)
        at java.base/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:240)
        at java.base/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:74)
        at java.base/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:103)
        at java.base/java.io.InputStream.read(InputStream.java:217)
        at java.base/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:82)
        at com.adobe.testing.s3mock.util.AwsChunkedDecodingInputStream.read(AwsChunkedDecodingInputStream.java:91)
        at java.base/java.io.InputStream.read(InputStream.java:293)
        at java.base/java.io.InputStream.read(InputStream.java:217)
        at com.adobe.testing.s3mock.store.ObjectStore.inputStreamToFile(ObjectStore.java:370)
        at com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:135)
        at com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
        at java.base/java.lang.reflect.Method.invoke(Method.java:578)
        at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.Server.handle(Server.java:563)
        at org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base/java.lang.Thread.run(Thread.java:1589)
   3) Thread[id=64, name=qtp1737344236-64, state=BLOCKED, group=TGRP-S3BackupRepositoryTest]
        at com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:133)
        at com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
        at java.base/java.lang.reflect.Method.invoke(Method.java:578)
        at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.Server.handle(Server.java:563)
        at org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base/java.lang.Thread.run(Thread.java:1589)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 3 threads leaked from SUITE scope at org.apache.solr.s3.S3BackupRepositoryTest: 
   1) Thread[id=58, name=qtp1737344236-58, state=BLOCKED, group=TGRP-S3BackupRepositoryTest]
        at com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:133)
        at com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
        at java.base/java.lang.reflect.Method.invoke(Method.java:578)
        at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.Server.handle(Server.java:563)
        at org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base/java.lang.Thread.run(Thread.java:1589)
   2) Thread[id=66, name=qtp1737344236-66, state=RUNNABLE, group=TGRP-S3BackupRepositoryTest]
        at java.base/sun.nio.ch.FileDispatcherImpl.read0(Native Method)
        at java.base/sun.nio.ch.FileDispatcherImpl.read(FileDispatcherImpl.java:48)
        at java.base/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:339)
        at java.base/sun.nio.ch.IOUtil.read(IOUtil.java:305)
        at java.base/sun.nio.ch.IOUtil.read(IOUtil.java:282)
        at java.base/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:240)
        at java.base/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:74)
        at java.base/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:103)
        at java.base/java.io.InputStream.read(InputStream.java:217)
        at java.base/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:82)
        at com.adobe.testing.s3mock.util.AwsChunkedDecodingInputStream.read(AwsChunkedDecodingInputStream.java:91)
        at java.base/java.io.InputStream.read(InputStream.java:293)
        at java.base/java.io.InputStream.read(InputStream.java:217)
        at com.adobe.testing.s3mock.store.ObjectStore.inputStreamToFile(ObjectStore.java:370)
        at com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:135)
        at com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
        at java.base/java.lang.reflect.Method.invoke(Method.java:578)
        at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.Server.handle(Server.java:563)
        at org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base/java.lang.Thread.run(Thread.java:1589)
   3) Thread[id=64, name=qtp1737344236-64, state=BLOCKED, group=TGRP-S3BackupRepositoryTest]
        at com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:133)
        at com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
        at java.base/java.lang.reflect.Method.invoke(Method.java:578)
        at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.Server.handle(Server.java:563)
        at org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base/java.lang.Thread.run(Thread.java:1589)
	at __randomizedtesting.SeedInfo.seed([AF876C82CAA2922F]:0)


FAILED:  org.apache.solr.s3.S3BackupRepositoryTest.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=58, name=qtp1737344236-58, state=BLOCKED, group=TGRP-S3BackupRepositoryTest]
        at com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:133)
        at com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
        at java.base/java.lang.reflect.Method.invoke(Method.java:578)
        at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.Server.handle(Server.java:563)
        at org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base/java.lang.Thread.run(Thread.java:1589)
   2) Thread[id=66, name=qtp1737344236-66, state=RUNNABLE, group=TGRP-S3BackupRepositoryTest]
        at java.base/sun.nio.ch.FileDispatcherImpl.read0(Native Method)
        at java.base/sun.nio.ch.FileDispatcherImpl.read(FileDispatcherImpl.java:48)
        at java.base/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:339)
        at java.base/sun.nio.ch.IOUtil.read(IOUtil.java:305)
        at java.base/sun.nio.ch.IOUtil.read(IOUtil.java:282)
        at java.base/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:240)
        at java.base/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:74)
        at java.base/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:103)
        at java.base/java.io.InputStream.read(InputStream.java:217)
        at java.base/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:82)
        at com.adobe.testing.s3mock.util.AwsChunkedDecodingInputStream.read(AwsChunkedDecodingInputStream.java:91)
        at java.base/java.io.InputStream.read(InputStream.java:293)
        at java.base/java.io.InputStream.read(InputStream.java:217)
        at com.adobe.testing.s3mock.store.ObjectStore.inputStreamToFile(ObjectStore.java:370)
        at com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:135)
        at com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
        at java.base/java.lang.reflect.Method.invoke(Method.java:578)
        at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.Server.handle(Server.java:563)
        at org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base/java.lang.Thread.run(Thread.java:1589)
   3) Thread[id=64, name=qtp1737344236-64, state=BLOCKED, group=TGRP-S3BackupRepositoryTest]
        at com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:133)
        at com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
        at java.base/java.lang.reflect.Method.invoke(Method.java:578)
        at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.Server.handle(Server.java:563)
        at org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base/java.lang.Thread.run(Thread.java:1589)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=58, name=qtp1737344236-58, state=BLOCKED, group=TGRP-S3BackupRepositoryTest]
        at com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:133)
        at com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
        at java.base/java.lang.reflect.Method.invoke(Method.java:578)
        at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.Server.handle(Server.java:563)
        at org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base/java.lang.Thread.run(Thread.java:1589)
   2) Thread[id=66, name=qtp1737344236-66, state=RUNNABLE, group=TGRP-S3BackupRepositoryTest]
        at java.base/sun.nio.ch.FileDispatcherImpl.read0(Native Method)
        at java.base/sun.nio.ch.FileDispatcherImpl.read(FileDispatcherImpl.java:48)
        at java.base/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:339)
        at java.base/sun.nio.ch.IOUtil.read(IOUtil.java:305)
        at java.base/sun.nio.ch.IOUtil.read(IOUtil.java:282)
        at java.base/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:240)
        at java.base/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:74)
        at java.base/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:103)
        at java.base/java.io.InputStream.read(InputStream.java:217)
        at java.base/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:82)
        at com.adobe.testing.s3mock.util.AwsChunkedDecodingInputStream.read(AwsChunkedDecodingInputStream.java:91)
        at java.base/java.io.InputStream.read(InputStream.java:293)
        at java.base/java.io.InputStream.read(InputStream.java:217)
        at com.adobe.testing.s3mock.store.ObjectStore.inputStreamToFile(ObjectStore.java:370)
        at com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:135)
        at com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
        at java.base/java.lang.reflect.Method.invoke(Method.java:578)
        at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.Server.handle(Server.java:563)
        at org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base/java.lang.Thread.run(Thread.java:1589)
   3) Thread[id=64, name=qtp1737344236-64, state=BLOCKED, group=TGRP-S3BackupRepositoryTest]
        at com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:133)
        at com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
        at java.base/java.lang.reflect.Method.invoke(Method.java:578)
        at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.Server.handle(Server.java:563)
        at org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base/java.lang.Thread.run(Thread.java:1589)
	at __randomizedtesting.SeedInfo.seed([AF876C82CAA2922F]:0)

[JENKINS] Solr-main-MacOSX (64bit/hotspot/jdk-19) - Build # 2600 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/2600/
Java: 64bit/hotspot/jdk-19 -XX:+UseCompressedOops -XX:+UseSerialGC

3 tests failed.
FAILED:  org.apache.solr.s3.S3BackupRepositoryTest.testCopyFiles

Error Message:
software.amazon.awssdk.core.exception.SdkClientException: Unable to execute HTTP request: Read timed out

Stack Trace:
software.amazon.awssdk.core.exception.SdkClientException: Unable to execute HTTP request: Read timed out
	at __randomizedtesting.SeedInfo.seed([6C49F5F7A8C84293:A295C1C382766FE3]:0)
	at app//software.amazon.awssdk.core.exception.SdkClientException$BuilderImpl.build(SdkClientException.java:111)
	at app//software.amazon.awssdk.core.exception.SdkClientException.create(SdkClientException.java:47)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.utils.RetryableStageHelper.setLastException(RetryableStageHelper.java:223)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.RetryableStage.execute(RetryableStage.java:83)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.RetryableStage.execute(RetryableStage.java:36)
	at app//software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206)
	at app//software.amazon.awssdk.core.internal.http.StreamManagingStage.execute(StreamManagingStage.java:56)
	at app//software.amazon.awssdk.core.internal.http.StreamManagingStage.execute(StreamManagingStage.java:36)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallTimeoutTrackingStage.executeWithTimer(ApiCallTimeoutTrackingStage.java:80)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallTimeoutTrackingStage.execute(ApiCallTimeoutTrackingStage.java:60)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallTimeoutTrackingStage.execute(ApiCallTimeoutTrackingStage.java:42)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallMetricCollectionStage.execute(ApiCallMetricCollectionStage.java:48)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallMetricCollectionStage.execute(ApiCallMetricCollectionStage.java:31)
	at app//software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206)
	at app//software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.ExecutionFailureExceptionReportingStage.execute(ExecutionFailureExceptionReportingStage.java:37)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.ExecutionFailureExceptionReportingStage.execute(ExecutionFailureExceptionReportingStage.java:26)
	at app//software.amazon.awssdk.core.internal.http.AmazonSyncHttpClient$RequestExecutionBuilderImpl.execute(AmazonSyncHttpClient.java:193)
	at app//software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.invoke(BaseSyncClientHandler.java:103)
	at app//software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.doExecute(BaseSyncClientHandler.java:171)
	at app//software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.lambda$execute$1(BaseSyncClientHandler.java:82)
	at app//software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.measureApiCallSuccess(BaseSyncClientHandler.java:179)
	at app//software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.execute(BaseSyncClientHandler.java:76)
	at app//software.amazon.awssdk.core.client.handler.SdkSyncClientHandler.execute(SdkSyncClientHandler.java:45)
	at app//software.amazon.awssdk.awscore.client.handler.AwsSyncClientHandler.execute(AwsSyncClientHandler.java:56)
	at app//software.amazon.awssdk.services.s3.DefaultS3Client.putObject(DefaultS3Client.java:9326)
	at app//software.amazon.awssdk.services.s3.S3Client.putObject(S3Client.java:17280)
	at app//org.apache.solr.s3.S3BackupRepositoryTest.pushObject(S3BackupRepositoryTest.java:328)
	at app//org.apache.solr.s3.S3BackupRepositoryTest.doTestCopyFileTo(S3BackupRepositoryTest.java:214)
	at app//org.apache.solr.s3.S3BackupRepositoryTest.testCopyFiles(S3BackupRepositoryTest.java:173)
	at java.base@19/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
	at java.base@19/java.lang.reflect.Method.invoke(Method.java:578)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at app//org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:54)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at app//com.adobe.testing.s3mock.junit4.S3MockRule$1.evaluate(S3MockRule.java:66)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
	at java.base@19/java.lang.Thread.run(Thread.java:1589)
	Suppressed: software.amazon.awssdk.core.exception.SdkClientException: Request attempt 1 failure: Unable to execute HTTP request: Read timed out
	Suppressed: software.amazon.awssdk.core.exception.SdkClientException: Request attempt 2 failure: Unable to execute HTTP request: Read timed out
	Suppressed: software.amazon.awssdk.core.exception.SdkClientException: Request attempt 3 failure: Unable to execute HTTP request: Read timed out
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.base/sun.nio.ch.NioSocketImpl.timedRead(NioSocketImpl.java:273)
	at java.base/sun.nio.ch.NioSocketImpl.implRead(NioSocketImpl.java:299)
	at java.base/sun.nio.ch.NioSocketImpl.read(NioSocketImpl.java:340)
	at java.base/sun.nio.ch.NioSocketImpl$1.read(NioSocketImpl.java:789)
	at java.base/java.net.Socket$SocketInputStream.read(Socket.java:1025)
	at java.base/java.io.BufferedInputStream.fill(BufferedInputStream.java:255)
	at java.base/java.io.BufferedInputStream.read1(BufferedInputStream.java:310)
	at java.base/java.io.BufferedInputStream.implRead(BufferedInputStream.java:382)
	at java.base/java.io.BufferedInputStream.read(BufferedInputStream.java:361)
	at java.base/sun.net.www.http.HttpClient.parseHTTPHeader(HttpClient.java:827)
	at java.base/sun.net.www.http.HttpClient.parseHTTP(HttpClient.java:759)
	at java.base/sun.net.www.protocol.http.HttpURLConnection.getInputStream0(HttpURLConnection.java:1684)
	at java.base/sun.net.www.protocol.http.HttpURLConnection.getInputStream(HttpURLConnection.java:1585)
	at java.base/java.net.HttpURLConnection.getResponseCode(HttpURLConnection.java:529)
	at software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient$RequestCallable.getResponseCodeSafely(UrlConnectionHttpClient.java:419)
	at software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient$RequestCallable.call(UrlConnectionHttpClient.java:302)
	at software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient$RequestCallable.call(UrlConnectionHttpClient.java:269)
	at software.amazon.awssdk.core.internal.util.MetricUtils.measureDurationUnsafe(MetricUtils.java:63)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.MakeHttpRequestStage.executeHttpRequest(MakeHttpRequestStage.java:77)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.MakeHttpRequestStage.execute(MakeHttpRequestStage.java:56)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.MakeHttpRequestStage.execute(MakeHttpRequestStage.java:39)
	at software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206)
	at software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206)
	at software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206)
	at software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallAttemptTimeoutTrackingStage.execute(ApiCallAttemptTimeoutTrackingStage.java:73)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallAttemptTimeoutTrackingStage.execute(ApiCallAttemptTimeoutTrackingStage.java:42)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.TimeoutExceptionHandlingStage.execute(TimeoutExceptionHandlingStage.java:78)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.TimeoutExceptionHandlingStage.execute(TimeoutExceptionHandlingStage.java:40)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallAttemptMetricCollectionStage.execute(ApiCallAttemptMetricCollectionStage.java:50)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallAttemptMetricCollectionStage.execute(ApiCallAttemptMetricCollectionStage.java:36)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.RetryableStage.execute(RetryableStage.java:81)
	... 73 more


FAILED:  org.apache.solr.s3.S3BackupRepositoryTest.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 4 threads leaked from SUITE scope at org.apache.solr.s3.S3BackupRepositoryTest: 
   1) Thread[id=28, name=qtp405541822-28, state=RUNNABLE, group=TGRP-S3BackupRepositoryTest]
        at java.base/sun.nio.ch.FileDispatcherImpl.read0(Native Method)
        at java.base/sun.nio.ch.FileDispatcherImpl.read(FileDispatcherImpl.java:48)
        at java.base/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:339)
        at java.base/sun.nio.ch.IOUtil.read(IOUtil.java:305)
        at java.base/sun.nio.ch.IOUtil.read(IOUtil.java:282)
        at java.base/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:240)
        at java.base/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:74)
        at java.base/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:103)
        at java.base/java.io.InputStream.read(InputStream.java:217)
        at java.base/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:82)
        at com.adobe.testing.s3mock.util.AwsChunkedDecodingInputStream.read(AwsChunkedDecodingInputStream.java:91)
        at java.base/java.io.InputStream.read(InputStream.java:293)
        at java.base/java.io.InputStream.read(InputStream.java:217)
        at com.adobe.testing.s3mock.store.ObjectStore.inputStreamToFile(ObjectStore.java:370)
        at com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:135)
        at com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
        at java.base/java.lang.reflect.Method.invoke(Method.java:578)
        at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.Server.handle(Server.java:563)
        at org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.runTask(AdaptiveExecutionStrategy.java:416)
        at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.consumeTask(AdaptiveExecutionStrategy.java:385)
        at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:272)
        at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.lambda$new$0(AdaptiveExecutionStrategy.java:140)
        at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:411)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base/java.lang.Thread.run(Thread.java:1589)
   2) Thread[id=55, name=qtp405541822-55, state=BLOCKED, group=TGRP-S3BackupRepositoryTest]
        at com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:133)
        at com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
        at java.base/java.lang.reflect.Method.invoke(Method.java:578)
        at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.Server.handle(Server.java:563)
        at org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base/java.lang.Thread.run(Thread.java:1589)
   3) Thread[id=51, name=qtp405541822-51, state=BLOCKED, group=TGRP-S3BackupRepositoryTest]
        at com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:133)
        at com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
        at java.base/java.lang.reflect.Method.invoke(Method.java:578)
        at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.Server.handle(Server.java:563)
        at org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base/java.lang.Thread.run(Thread.java:1589)
   4) Thread[id=57, name=qtp405541822-57, state=BLOCKED, group=TGRP-S3BackupRepositoryTest]
        at com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:133)
        at com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
        at java.base/java.lang.reflect.Method.invoke(Method.java:578)
        at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.Server.handle(Server.java:563)
        at org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base/java.lang.Thread.run(Thread.java:1589)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 4 threads leaked from SUITE scope at org.apache.solr.s3.S3BackupRepositoryTest: 
   1) Thread[id=28, name=qtp405541822-28, state=RUNNABLE, group=TGRP-S3BackupRepositoryTest]
        at java.base/sun.nio.ch.FileDispatcherImpl.read0(Native Method)
        at java.base/sun.nio.ch.FileDispatcherImpl.read(FileDispatcherImpl.java:48)
        at java.base/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:339)
        at java.base/sun.nio.ch.IOUtil.read(IOUtil.java:305)
        at java.base/sun.nio.ch.IOUtil.read(IOUtil.java:282)
        at java.base/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:240)
        at java.base/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:74)
        at java.base/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:103)
        at java.base/java.io.InputStream.read(InputStream.java:217)
        at java.base/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:82)
        at com.adobe.testing.s3mock.util.AwsChunkedDecodingInputStream.read(AwsChunkedDecodingInputStream.java:91)
        at java.base/java.io.InputStream.read(InputStream.java:293)
        at java.base/java.io.InputStream.read(InputStream.java:217)
        at com.adobe.testing.s3mock.store.ObjectStore.inputStreamToFile(ObjectStore.java:370)
        at com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:135)
        at com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
        at java.base/java.lang.reflect.Method.invoke(Method.java:578)
        at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.Server.handle(Server.java:563)
        at org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.runTask(AdaptiveExecutionStrategy.java:416)
        at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.consumeTask(AdaptiveExecutionStrategy.java:385)
        at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:272)
        at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.lambda$new$0(AdaptiveExecutionStrategy.java:140)
        at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:411)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base/java.lang.Thread.run(Thread.java:1589)
   2) Thread[id=55, name=qtp405541822-55, state=BLOCKED, group=TGRP-S3BackupRepositoryTest]
        at com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:133)
        at com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
        at java.base/java.lang.reflect.Method.invoke(Method.java:578)
        at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.Server.handle(Server.java:563)
        at org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base/java.lang.Thread.run(Thread.java:1589)
   3) Thread[id=51, name=qtp405541822-51, state=BLOCKED, group=TGRP-S3BackupRepositoryTest]
        at com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:133)
        at com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
        at java.base/java.lang.reflect.Method.invoke(Method.java:578)
        at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.Server.handle(Server.java:563)
        at org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base/java.lang.Thread.run(Thread.java:1589)
   4) Thread[id=57, name=qtp405541822-57, state=BLOCKED, group=TGRP-S3BackupRepositoryTest]
        at com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:133)
        at com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
        at java.base/java.lang.reflect.Method.invoke(Method.java:578)
        at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.Server.handle(Server.java:563)
        at org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base/java.lang.Thread.run(Thread.java:1589)
	at __randomizedtesting.SeedInfo.seed([6C49F5F7A8C84293]:0)


FAILED:  org.apache.solr.s3.S3BackupRepositoryTest.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=28, name=qtp405541822-28, state=RUNNABLE, group=TGRP-S3BackupRepositoryTest]
        at java.base/sun.nio.ch.FileDispatcherImpl.read0(Native Method)
        at java.base/sun.nio.ch.FileDispatcherImpl.read(FileDispatcherImpl.java:48)
        at java.base/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:339)
        at java.base/sun.nio.ch.IOUtil.read(IOUtil.java:305)
        at java.base/sun.nio.ch.IOUtil.read(IOUtil.java:282)
        at java.base/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:240)
        at java.base/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:74)
        at java.base/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:103)
        at java.base/java.io.InputStream.read(InputStream.java:217)
        at java.base/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:82)
        at com.adobe.testing.s3mock.util.AwsChunkedDecodingInputStream.read(AwsChunkedDecodingInputStream.java:91)
        at java.base/java.io.InputStream.read(InputStream.java:293)
        at java.base/java.io.InputStream.read(InputStream.java:217)
        at com.adobe.testing.s3mock.store.ObjectStore.inputStreamToFile(ObjectStore.java:370)
        at com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:135)
        at com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
        at java.base/java.lang.reflect.Method.invoke(Method.java:578)
        at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.Server.handle(Server.java:563)
        at org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.runTask(AdaptiveExecutionStrategy.java:416)
        at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.consumeTask(AdaptiveExecutionStrategy.java:385)
        at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:272)
        at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.lambda$new$0(AdaptiveExecutionStrategy.java:140)
        at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:411)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base/java.lang.Thread.run(Thread.java:1589)
   2) Thread[id=55, name=qtp405541822-55, state=BLOCKED, group=TGRP-S3BackupRepositoryTest]
        at com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:133)
        at com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
        at java.base/java.lang.reflect.Method.invoke(Method.java:578)
        at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.Server.handle(Server.java:563)
        at org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base/java.lang.Thread.run(Thread.java:1589)
   3) Thread[id=51, name=qtp405541822-51, state=BLOCKED, group=TGRP-S3BackupRepositoryTest]
        at com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:133)
        at com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
        at java.base/java.lang.reflect.Method.invoke(Method.java:578)
        at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.Server.handle(Server.java:563)
        at org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base/java.lang.Thread.run(Thread.java:1589)
   4) Thread[id=57, name=qtp405541822-57, state=BLOCKED, group=TGRP-S3BackupRepositoryTest]
        at com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:133)
        at com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
        at java.base/java.lang.reflect.Method.invoke(Method.java:578)
        at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.Server.handle(Server.java:563)
        at org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base/java.lang.Thread.run(Thread.java:1589)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=28, name=qtp405541822-28, state=RUNNABLE, group=TGRP-S3BackupRepositoryTest]
        at java.base/sun.nio.ch.FileDispatcherImpl.read0(Native Method)
        at java.base/sun.nio.ch.FileDispatcherImpl.read(FileDispatcherImpl.java:48)
        at java.base/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:339)
        at java.base/sun.nio.ch.IOUtil.read(IOUtil.java:305)
        at java.base/sun.nio.ch.IOUtil.read(IOUtil.java:282)
        at java.base/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:240)
        at java.base/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:74)
        at java.base/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:103)
        at java.base/java.io.InputStream.read(InputStream.java:217)
        at java.base/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:82)
        at com.adobe.testing.s3mock.util.AwsChunkedDecodingInputStream.read(AwsChunkedDecodingInputStream.java:91)
        at java.base/java.io.InputStream.read(InputStream.java:293)
        at java.base/java.io.InputStream.read(InputStream.java:217)
        at com.adobe.testing.s3mock.store.ObjectStore.inputStreamToFile(ObjectStore.java:370)
        at com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:135)
        at com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
        at java.base/java.lang.reflect.Method.invoke(Method.java:578)
        at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.Server.handle(Server.java:563)
        at org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.runTask(AdaptiveExecutionStrategy.java:416)
        at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.consumeTask(AdaptiveExecutionStrategy.java:385)
        at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:272)
        at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.lambda$new$0(AdaptiveExecutionStrategy.java:140)
        at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:411)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base/java.lang.Thread.run(Thread.java:1589)
   2) Thread[id=55, name=qtp405541822-55, state=BLOCKED, group=TGRP-S3BackupRepositoryTest]
        at com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:133)
        at com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
        at java.base/java.lang.reflect.Method.invoke(Method.java:578)
        at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.Server.handle(Server.java:563)
        at org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base/java.lang.Thread.run(Thread.java:1589)
   3) Thread[id=51, name=qtp405541822-51, state=BLOCKED, group=TGRP-S3BackupRepositoryTest]
        at com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:133)
        at com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
        at java.base/java.lang.reflect.Method.invoke(Method.java:578)
        at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.Server.handle(Server.java:563)
        at org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base/java.lang.Thread.run(Thread.java:1589)
   4) Thread[id=57, name=qtp405541822-57, state=BLOCKED, group=TGRP-S3BackupRepositoryTest]
        at com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:133)
        at com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
        at java.base/java.lang.reflect.Method.invoke(Method.java:578)
        at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.Server.handle(Server.java:563)
        at org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base/java.lang.Thread.run(Thread.java:1589)
	at __randomizedtesting.SeedInfo.seed([6C49F5F7A8C84293]:0)

[JENKINS] Solr-main-MacOSX (64bit/hotspot/jdk-19) - Build # 2599 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/2599/
Java: 64bit/hotspot/jdk-19 -XX:-UseCompressedOops -XX:+UseSerialGC

5 tests failed.
FAILED:  org.apache.solr.cloud.BasicDistributedZk2Test.test

Error Message:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:55873/b_dv/rq/collection1: No registered leader was found after waiting for 4000ms , collection: collection1 slice: shard1 saw state=DocCollection(collection1//collections/collection1/state.json/13 )={
  "pullReplicas":"0",
  "configName":"conf1",
  "replicationFactor":"1",
  "shards":{
    "shard1":{
      "range":"80000000-ffffffff",
      "replicas":{
        "core_node4":{
          "node_name":"127.0.0.1:55858_b_dv%2Frq",
          "base_url":"http://127.0.0.1:55858/b_dv/rq",
          "core":"collection1_shard1_replica_n2",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node7":{
          "node_name":"127.0.0.1:55873_b_dv%2Frq",
          "base_url":"http://127.0.0.1:55873/b_dv/rq",
          "core":"collection1_shard1_replica_n5",
          "state":"active",
          "type":"NRT",
          "force_set_state":"false"}},
      "state":"active"},
    "shard2":{
      "range":"0-7fffffff",
      "replicas":{
        "core_node3":{
          "core":"collection1_shard2_replica_n1",
          "leader":"true",
          "node_name":"127.0.0.1:55850_b_dv%2Frq",
          "base_url":"http://127.0.0.1:55850/b_dv/rq",
          "state":"active",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node8":{
          "node_name":"127.0.0.1:55865_b_dv%2Frq",
          "base_url":"http://127.0.0.1:55865/b_dv/rq",
          "core":"collection1_shard2_replica_n6",
          "state":"active",
          "type":"NRT",
          "force_set_state":"false"}},
      "state":"active"}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"1",
  "tlogReplicas":"0"} with live_nodes=[127.0.0.1:55850_b_dv%2Frq, 127.0.0.1:55795_b_dv%2Frq, 127.0.0.1:55873_b_dv%2Frq, 127.0.0.1:55865_b_dv%2Frq]

Stack Trace:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:55873/b_dv/rq/collection1: No registered leader was found after waiting for 4000ms , collection: collection1 slice: shard1 saw state=DocCollection(collection1//collections/collection1/state.json/13 )={
  "pullReplicas":"0",
  "configName":"conf1",
  "replicationFactor":"1",
  "shards":{
    "shard1":{
      "range":"80000000-ffffffff",
      "replicas":{
        "core_node4":{
          "node_name":"127.0.0.1:55858_b_dv%2Frq",
          "base_url":"http://127.0.0.1:55858/b_dv/rq",
          "core":"collection1_shard1_replica_n2",
          "state":"down",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node7":{
          "node_name":"127.0.0.1:55873_b_dv%2Frq",
          "base_url":"http://127.0.0.1:55873/b_dv/rq",
          "core":"collection1_shard1_replica_n5",
          "state":"active",
          "type":"NRT",
          "force_set_state":"false"}},
      "state":"active"},
    "shard2":{
      "range":"0-7fffffff",
      "replicas":{
        "core_node3":{
          "core":"collection1_shard2_replica_n1",
          "leader":"true",
          "node_name":"127.0.0.1:55850_b_dv%2Frq",
          "base_url":"http://127.0.0.1:55850/b_dv/rq",
          "state":"active",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node8":{
          "node_name":"127.0.0.1:55865_b_dv%2Frq",
          "base_url":"http://127.0.0.1:55865/b_dv/rq",
          "core":"collection1_shard2_replica_n6",
          "state":"active",
          "type":"NRT",
          "force_set_state":"false"}},
      "state":"active"}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"1",
  "tlogReplicas":"0"} with live_nodes=[127.0.0.1:55850_b_dv%2Frq, 127.0.0.1:55795_b_dv%2Frq, 127.0.0.1:55873_b_dv%2Frq, 127.0.0.1:55865_b_dv%2Frq]
	at __randomizedtesting.SeedInfo.seed([33C3CD6B21B0816:8B68030C1CE765EE]:0)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:719)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:386)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:352)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1134)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:767)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:234)
	at app//org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:466)
	at app//org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:442)
	at app//org.apache.solr.cloud.AbstractFullDistribZkTestBase.commit(AbstractFullDistribZkTestBase.java:2020)
	at app//org.apache.solr.cloud.AbstractBasicDistributedZk2TestBase.bringDownShardIndexSomeDocsAndRecover(AbstractBasicDistributedZk2TestBase.java:286)
	at app//org.apache.solr.cloud.AbstractBasicDistributedZk2TestBase.test(AbstractBasicDistributedZk2TestBase.java:116)
	at app//org.apache.solr.cloud.BasicDistributedZk2Test.test(BasicDistributedZk2Test.java:34)
	at java.base@19/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
	at java.base@19/java.lang.reflect.Method.invoke(Method.java:578)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1163)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1134)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
	at java.base@19/java.lang.Thread.run(Thread.java:1589)


FAILED:  org.apache.solr.cloud.PeerSyncReplicationTest.test

Error Message:
java.lang.AssertionError: expected:<154> but was:<152>

Stack Trace:
java.lang.AssertionError: expected:<154> but was:<152>
	at __randomizedtesting.SeedInfo.seed([33C3CD6B21B0816:8B68030C1CE765EE]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.junit.Assert.failNotEquals(Assert.java:835)
	at org.junit.Assert.assertEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:633)
	at org.apache.solr.cloud.PeerSyncReplicationTest.test(PeerSyncReplicationTest.java:145)
	at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
	at java.base/java.lang.reflect.Method.invoke(Method.java:578)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1163)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1134)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
	at java.base/java.lang.Thread.run(Thread.java:1589)


FAILED:  org.apache.solr.s3.S3BackupRepositoryTest.testCopyFiles

Error Message:
software.amazon.awssdk.core.exception.SdkClientException: Unable to execute HTTP request: Read timed out

Stack Trace:
software.amazon.awssdk.core.exception.SdkClientException: Unable to execute HTTP request: Read timed out
	at __randomizedtesting.SeedInfo.seed([33C3CD6B21B0816:CDE008E298A52566]:0)
	at app//software.amazon.awssdk.core.exception.SdkClientException$BuilderImpl.build(SdkClientException.java:111)
	at app//software.amazon.awssdk.core.exception.SdkClientException.create(SdkClientException.java:47)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.utils.RetryableStageHelper.setLastException(RetryableStageHelper.java:223)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.RetryableStage.execute(RetryableStage.java:83)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.RetryableStage.execute(RetryableStage.java:36)
	at app//software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206)
	at app//software.amazon.awssdk.core.internal.http.StreamManagingStage.execute(StreamManagingStage.java:56)
	at app//software.amazon.awssdk.core.internal.http.StreamManagingStage.execute(StreamManagingStage.java:36)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallTimeoutTrackingStage.executeWithTimer(ApiCallTimeoutTrackingStage.java:80)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallTimeoutTrackingStage.execute(ApiCallTimeoutTrackingStage.java:60)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallTimeoutTrackingStage.execute(ApiCallTimeoutTrackingStage.java:42)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallMetricCollectionStage.execute(ApiCallMetricCollectionStage.java:48)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallMetricCollectionStage.execute(ApiCallMetricCollectionStage.java:31)
	at app//software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206)
	at app//software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.ExecutionFailureExceptionReportingStage.execute(ExecutionFailureExceptionReportingStage.java:37)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.ExecutionFailureExceptionReportingStage.execute(ExecutionFailureExceptionReportingStage.java:26)
	at app//software.amazon.awssdk.core.internal.http.AmazonSyncHttpClient$RequestExecutionBuilderImpl.execute(AmazonSyncHttpClient.java:193)
	at app//software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.invoke(BaseSyncClientHandler.java:103)
	at app//software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.doExecute(BaseSyncClientHandler.java:171)
	at app//software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.lambda$execute$1(BaseSyncClientHandler.java:82)
	at app//software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.measureApiCallSuccess(BaseSyncClientHandler.java:179)
	at app//software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.execute(BaseSyncClientHandler.java:76)
	at app//software.amazon.awssdk.core.client.handler.SdkSyncClientHandler.execute(SdkSyncClientHandler.java:45)
	at app//software.amazon.awssdk.awscore.client.handler.AwsSyncClientHandler.execute(AwsSyncClientHandler.java:56)
	at app//software.amazon.awssdk.services.s3.DefaultS3Client.putObject(DefaultS3Client.java:9326)
	at app//software.amazon.awssdk.services.s3.S3Client.putObject(S3Client.java:17280)
	at app//org.apache.solr.s3.S3BackupRepositoryTest.pushObject(S3BackupRepositoryTest.java:328)
	at app//org.apache.solr.s3.S3BackupRepositoryTest.doTestCopyFileTo(S3BackupRepositoryTest.java:214)
	at app//org.apache.solr.s3.S3BackupRepositoryTest.testCopyFiles(S3BackupRepositoryTest.java:173)
	at java.base@19/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
	at java.base@19/java.lang.reflect.Method.invoke(Method.java:578)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at app//org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:54)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at app//com.adobe.testing.s3mock.junit4.S3MockRule$1.evaluate(S3MockRule.java:66)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
	at java.base@19/java.lang.Thread.run(Thread.java:1589)
	Suppressed: software.amazon.awssdk.core.exception.SdkClientException: Request attempt 1 failure: Unable to execute HTTP request: Read timed out
	Suppressed: software.amazon.awssdk.core.exception.SdkClientException: Request attempt 2 failure: Unable to execute HTTP request: Read timed out
	Suppressed: software.amazon.awssdk.core.exception.SdkClientException: Request attempt 3 failure: Unable to execute HTTP request: Read timed out
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.base/sun.nio.ch.NioSocketImpl.timedRead(NioSocketImpl.java:273)
	at java.base/sun.nio.ch.NioSocketImpl.implRead(NioSocketImpl.java:299)
	at java.base/sun.nio.ch.NioSocketImpl.read(NioSocketImpl.java:340)
	at java.base/sun.nio.ch.NioSocketImpl$1.read(NioSocketImpl.java:789)
	at java.base/java.net.Socket$SocketInputStream.read(Socket.java:1025)
	at java.base/java.io.BufferedInputStream.fill(BufferedInputStream.java:255)
	at java.base/java.io.BufferedInputStream.read1(BufferedInputStream.java:310)
	at java.base/java.io.BufferedInputStream.implRead(BufferedInputStream.java:382)
	at java.base/java.io.BufferedInputStream.read(BufferedInputStream.java:361)
	at java.base/sun.net.www.http.HttpClient.parseHTTPHeader(HttpClient.java:827)
	at java.base/sun.net.www.http.HttpClient.parseHTTP(HttpClient.java:759)
	at java.base/sun.net.www.protocol.http.HttpURLConnection.getInputStream0(HttpURLConnection.java:1684)
	at java.base/sun.net.www.protocol.http.HttpURLConnection.getInputStream(HttpURLConnection.java:1585)
	at java.base/java.net.HttpURLConnection.getResponseCode(HttpURLConnection.java:529)
	at software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient$RequestCallable.getResponseCodeSafely(UrlConnectionHttpClient.java:419)
	at software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient$RequestCallable.call(UrlConnectionHttpClient.java:302)
	at software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient$RequestCallable.call(UrlConnectionHttpClient.java:269)
	at software.amazon.awssdk.core.internal.util.MetricUtils.measureDurationUnsafe(MetricUtils.java:63)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.MakeHttpRequestStage.executeHttpRequest(MakeHttpRequestStage.java:77)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.MakeHttpRequestStage.execute(MakeHttpRequestStage.java:56)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.MakeHttpRequestStage.execute(MakeHttpRequestStage.java:39)
	at software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206)
	at software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206)
	at software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206)
	at software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallAttemptTimeoutTrackingStage.execute(ApiCallAttemptTimeoutTrackingStage.java:73)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallAttemptTimeoutTrackingStage.execute(ApiCallAttemptTimeoutTrackingStage.java:42)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.TimeoutExceptionHandlingStage.execute(TimeoutExceptionHandlingStage.java:78)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.TimeoutExceptionHandlingStage.execute(TimeoutExceptionHandlingStage.java:40)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallAttemptMetricCollectionStage.execute(ApiCallAttemptMetricCollectionStage.java:50)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallAttemptMetricCollectionStage.execute(ApiCallAttemptMetricCollectionStage.java:36)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.RetryableStage.execute(RetryableStage.java:81)
	... 73 more


FAILED:  org.apache.solr.s3.S3BackupRepositoryTest.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 3 threads leaked from SUITE scope at org.apache.solr.s3.S3BackupRepositoryTest: 
   1) Thread[id=54, name=qtp1178543237-54, state=BLOCKED, group=TGRP-S3BackupRepositoryTest]
        at com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:133)
        at com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
        at java.base/java.lang.reflect.Method.invoke(Method.java:578)
        at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.Server.handle(Server.java:563)
        at org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base/java.lang.Thread.run(Thread.java:1589)
   2) Thread[id=56, name=qtp1178543237-56, state=RUNNABLE, group=TGRP-S3BackupRepositoryTest]
        at java.base/sun.nio.ch.FileDispatcherImpl.read0(Native Method)
        at java.base/sun.nio.ch.FileDispatcherImpl.read(FileDispatcherImpl.java:48)
        at java.base/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:339)
        at java.base/sun.nio.ch.IOUtil.read(IOUtil.java:305)
        at java.base/sun.nio.ch.IOUtil.read(IOUtil.java:282)
        at java.base/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:240)
        at java.base/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:74)
        at java.base/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:103)
        at java.base/java.io.InputStream.read(InputStream.java:217)
        at java.base/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:82)
        at com.adobe.testing.s3mock.util.AwsChunkedDecodingInputStream.read(AwsChunkedDecodingInputStream.java:91)
        at java.base/java.io.InputStream.read(InputStream.java:293)
        at java.base/java.io.InputStream.read(InputStream.java:217)
        at com.adobe.testing.s3mock.store.ObjectStore.inputStreamToFile(ObjectStore.java:370)
        at com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:135)
        at com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
        at java.base/java.lang.reflect.Method.invoke(Method.java:578)
        at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.Server.handle(Server.java:563)
        at org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base/java.lang.Thread.run(Thread.java:1589)
   3) Thread[id=51, name=qtp1178543237-51, state=BLOCKED, group=TGRP-S3BackupRepositoryTest]
        at com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:133)
        at com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
        at java.base/java.lang.reflect.Method.invoke(Method.java:578)
        at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.Server.handle(Server.java:563)
        at org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base/java.lang.Thread.run(Thread.java:1589)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 3 threads leaked from SUITE scope at org.apache.solr.s3.S3BackupRepositoryTest: 
   1) Thread[id=54, name=qtp1178543237-54, state=BLOCKED, group=TGRP-S3BackupRepositoryTest]
        at com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:133)
        at com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
        at java.base/java.lang.reflect.Method.invoke(Method.java:578)
        at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.Server.handle(Server.java:563)
        at org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base/java.lang.Thread.run(Thread.java:1589)
   2) Thread[id=56, name=qtp1178543237-56, state=RUNNABLE, group=TGRP-S3BackupRepositoryTest]
        at java.base/sun.nio.ch.FileDispatcherImpl.read0(Native Method)
        at java.base/sun.nio.ch.FileDispatcherImpl.read(FileDispatcherImpl.java:48)
        at java.base/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:339)
        at java.base/sun.nio.ch.IOUtil.read(IOUtil.java:305)
        at java.base/sun.nio.ch.IOUtil.read(IOUtil.java:282)
        at java.base/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:240)
        at java.base/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:74)
        at java.base/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:103)
        at java.base/java.io.InputStream.read(InputStream.java:217)
        at java.base/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:82)
        at com.adobe.testing.s3mock.util.AwsChunkedDecodingInputStream.read(AwsChunkedDecodingInputStream.java:91)
        at java.base/java.io.InputStream.read(InputStream.java:293)
        at java.base/java.io.InputStream.read(InputStream.java:217)
        at com.adobe.testing.s3mock.store.ObjectStore.inputStreamToFile(ObjectStore.java:370)
        at com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:135)
        at com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
        at java.base/java.lang.reflect.Method.invoke(Method.java:578)
        at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.Server.handle(Server.java:563)
        at org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base/java.lang.Thread.run(Thread.java:1589)
   3) Thread[id=51, name=qtp1178543237-51, state=BLOCKED, group=TGRP-S3BackupRepositoryTest]
        at com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:133)
        at com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
        at java.base/java.lang.reflect.Method.invoke(Method.java:578)
        at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.Server.handle(Server.java:563)
        at org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base/java.lang.Thread.run(Thread.java:1589)
	at __randomizedtesting.SeedInfo.seed([33C3CD6B21B0816]:0)


FAILED:  org.apache.solr.s3.S3BackupRepositoryTest.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=54, name=qtp1178543237-54, state=BLOCKED, group=TGRP-S3BackupRepositoryTest]
        at com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:133)
        at com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
        at java.base/java.lang.reflect.Method.invoke(Method.java:578)
        at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.Server.handle(Server.java:563)
        at org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base/java.lang.Thread.run(Thread.java:1589)
   2) Thread[id=56, name=qtp1178543237-56, state=RUNNABLE, group=TGRP-S3BackupRepositoryTest]
        at java.base/sun.nio.ch.FileDispatcherImpl.read0(Native Method)
        at java.base/sun.nio.ch.FileDispatcherImpl.read(FileDispatcherImpl.java:48)
        at java.base/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:339)
        at java.base/sun.nio.ch.IOUtil.read(IOUtil.java:305)
        at java.base/sun.nio.ch.IOUtil.read(IOUtil.java:282)
        at java.base/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:240)
        at java.base/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:74)
        at java.base/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:103)
        at java.base/java.io.InputStream.read(InputStream.java:217)
        at java.base/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:82)
        at com.adobe.testing.s3mock.util.AwsChunkedDecodingInputStream.read(AwsChunkedDecodingInputStream.java:91)
        at java.base/java.io.InputStream.read(InputStream.java:293)
        at java.base/java.io.InputStream.read(InputStream.java:217)
        at com.adobe.testing.s3mock.store.ObjectStore.inputStreamToFile(ObjectStore.java:370)
        at com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:135)
        at com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
        at java.base/java.lang.reflect.Method.invoke(Method.java:578)
        at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.Server.handle(Server.java:563)
        at org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base/java.lang.Thread.run(Thread.java:1589)
   3) Thread[id=51, name=qtp1178543237-51, state=BLOCKED, group=TGRP-S3BackupRepositoryTest]
        at com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:133)
        at com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
        at java.base/java.lang.reflect.Method.invoke(Method.java:578)
        at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.Server.handle(Server.java:563)
        at org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base/java.lang.Thread.run(Thread.java:1589)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=54, name=qtp1178543237-54, state=BLOCKED, group=TGRP-S3BackupRepositoryTest]
        at com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:133)
        at com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
        at java.base/java.lang.reflect.Method.invoke(Method.java:578)
        at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.Server.handle(Server.java:563)
        at org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base/java.lang.Thread.run(Thread.java:1589)
   2) Thread[id=56, name=qtp1178543237-56, state=RUNNABLE, group=TGRP-S3BackupRepositoryTest]
        at java.base/sun.nio.ch.FileDispatcherImpl.read0(Native Method)
        at java.base/sun.nio.ch.FileDispatcherImpl.read(FileDispatcherImpl.java:48)
        at java.base/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:339)
        at java.base/sun.nio.ch.IOUtil.read(IOUtil.java:305)
        at java.base/sun.nio.ch.IOUtil.read(IOUtil.java:282)
        at java.base/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:240)
        at java.base/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:74)
        at java.base/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:103)
        at java.base/java.io.InputStream.read(InputStream.java:217)
        at java.base/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:82)
        at com.adobe.testing.s3mock.util.AwsChunkedDecodingInputStream.read(AwsChunkedDecodingInputStream.java:91)
        at java.base/java.io.InputStream.read(InputStream.java:293)
        at java.base/java.io.InputStream.read(InputStream.java:217)
        at com.adobe.testing.s3mock.store.ObjectStore.inputStreamToFile(ObjectStore.java:370)
        at com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:135)
        at com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
        at java.base/java.lang.reflect.Method.invoke(Method.java:578)
        at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.Server.handle(Server.java:563)
        at org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base/java.lang.Thread.run(Thread.java:1589)
   3) Thread[id=51, name=qtp1178543237-51, state=BLOCKED, group=TGRP-S3BackupRepositoryTest]
        at com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:133)
        at com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
        at java.base/java.lang.reflect.Method.invoke(Method.java:578)
        at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.Server.handle(Server.java:563)
        at org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base/java.lang.Thread.run(Thread.java:1589)
	at __randomizedtesting.SeedInfo.seed([33C3CD6B21B0816]:0)

[JENKINS] Solr-main-MacOSX (64bit/hotspot/jdk-19) - Build # 2598 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/2598/
Java: 64bit/hotspot/jdk-19 -XX:-UseCompressedOops -XX:+UseParallelGC

4 tests failed.
FAILED:  org.apache.solr.cloud.MoveReplicaTest.testFailedMove

Error Message:
java.lang.AssertionError: expected:<100> but was:<56>

Stack Trace:
java.lang.AssertionError: expected:<100> but was:<56>
	at __randomizedtesting.SeedInfo.seed([8DF81B0A5B0B9E2F:2735C8F8ECD84BFF]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.junit.Assert.failNotEquals(Assert.java:835)
	at org.junit.Assert.assertEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:633)
	at org.apache.solr.cloud.AbstractMoveReplicaTestBase.testFailedMove(AbstractMoveReplicaTestBase.java:331)
	at org.apache.solr.cloud.MoveReplicaTest.testFailedMove(MoveReplicaTest.java:37)
	at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
	at java.base/java.lang.reflect.Method.invoke(Method.java:578)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
	at java.base/java.lang.Thread.run(Thread.java:1589)


FAILED:  org.apache.solr.s3.S3BackupRepositoryTest.testCopyFiles

Error Message:
software.amazon.awssdk.core.exception.SdkClientException: Unable to execute HTTP request: Read timed out

Stack Trace:
software.amazon.awssdk.core.exception.SdkClientException: Unable to execute HTTP request: Read timed out
	at __randomizedtesting.SeedInfo.seed([8DF81B0A5B0B9E2F:43242F3E71B5B35F]:0)
	at app//software.amazon.awssdk.core.exception.SdkClientException$BuilderImpl.build(SdkClientException.java:111)
	at app//software.amazon.awssdk.core.exception.SdkClientException.create(SdkClientException.java:47)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.utils.RetryableStageHelper.setLastException(RetryableStageHelper.java:223)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.RetryableStage.execute(RetryableStage.java:83)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.RetryableStage.execute(RetryableStage.java:36)
	at app//software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206)
	at app//software.amazon.awssdk.core.internal.http.StreamManagingStage.execute(StreamManagingStage.java:56)
	at app//software.amazon.awssdk.core.internal.http.StreamManagingStage.execute(StreamManagingStage.java:36)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallTimeoutTrackingStage.executeWithTimer(ApiCallTimeoutTrackingStage.java:80)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallTimeoutTrackingStage.execute(ApiCallTimeoutTrackingStage.java:60)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallTimeoutTrackingStage.execute(ApiCallTimeoutTrackingStage.java:42)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallMetricCollectionStage.execute(ApiCallMetricCollectionStage.java:48)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallMetricCollectionStage.execute(ApiCallMetricCollectionStage.java:31)
	at app//software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206)
	at app//software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.ExecutionFailureExceptionReportingStage.execute(ExecutionFailureExceptionReportingStage.java:37)
	at app//software.amazon.awssdk.core.internal.http.pipeline.stages.ExecutionFailureExceptionReportingStage.execute(ExecutionFailureExceptionReportingStage.java:26)
	at app//software.amazon.awssdk.core.internal.http.AmazonSyncHttpClient$RequestExecutionBuilderImpl.execute(AmazonSyncHttpClient.java:193)
	at app//software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.invoke(BaseSyncClientHandler.java:103)
	at app//software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.doExecute(BaseSyncClientHandler.java:171)
	at app//software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.lambda$execute$1(BaseSyncClientHandler.java:82)
	at app//software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.measureApiCallSuccess(BaseSyncClientHandler.java:179)
	at app//software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.execute(BaseSyncClientHandler.java:76)
	at app//software.amazon.awssdk.core.client.handler.SdkSyncClientHandler.execute(SdkSyncClientHandler.java:45)
	at app//software.amazon.awssdk.awscore.client.handler.AwsSyncClientHandler.execute(AwsSyncClientHandler.java:56)
	at app//software.amazon.awssdk.services.s3.DefaultS3Client.putObject(DefaultS3Client.java:9326)
	at app//software.amazon.awssdk.services.s3.S3Client.putObject(S3Client.java:17280)
	at app//org.apache.solr.s3.S3BackupRepositoryTest.pushObject(S3BackupRepositoryTest.java:328)
	at app//org.apache.solr.s3.S3BackupRepositoryTest.doTestCopyFileTo(S3BackupRepositoryTest.java:214)
	at app//org.apache.solr.s3.S3BackupRepositoryTest.testCopyFiles(S3BackupRepositoryTest.java:173)
	at java.base@19/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
	at java.base@19/java.lang.reflect.Method.invoke(Method.java:578)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at app//org.junit.rules.ExternalResource$1.evaluate(ExternalResource.java:54)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at app//com.adobe.testing.s3mock.junit4.S3MockRule$1.evaluate(S3MockRule.java:66)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
	at java.base@19/java.lang.Thread.run(Thread.java:1589)
	Suppressed: software.amazon.awssdk.core.exception.SdkClientException: Request attempt 1 failure: Unable to execute HTTP request: Read timed out
	Suppressed: software.amazon.awssdk.core.exception.SdkClientException: Request attempt 2 failure: Unable to execute HTTP request: Read timed out
	Suppressed: software.amazon.awssdk.core.exception.SdkClientException: Request attempt 3 failure: Unable to execute HTTP request: Read timed out
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.base/sun.nio.ch.NioSocketImpl.timedRead(NioSocketImpl.java:273)
	at java.base/sun.nio.ch.NioSocketImpl.implRead(NioSocketImpl.java:299)
	at java.base/sun.nio.ch.NioSocketImpl.read(NioSocketImpl.java:340)
	at java.base/sun.nio.ch.NioSocketImpl$1.read(NioSocketImpl.java:789)
	at java.base/java.net.Socket$SocketInputStream.read(Socket.java:1025)
	at java.base/java.io.BufferedInputStream.fill(BufferedInputStream.java:255)
	at java.base/java.io.BufferedInputStream.read1(BufferedInputStream.java:310)
	at java.base/java.io.BufferedInputStream.implRead(BufferedInputStream.java:382)
	at java.base/java.io.BufferedInputStream.read(BufferedInputStream.java:361)
	at java.base/sun.net.www.http.HttpClient.parseHTTPHeader(HttpClient.java:827)
	at java.base/sun.net.www.http.HttpClient.parseHTTP(HttpClient.java:759)
	at java.base/sun.net.www.protocol.http.HttpURLConnection.getInputStream0(HttpURLConnection.java:1684)
	at java.base/sun.net.www.protocol.http.HttpURLConnection.getInputStream(HttpURLConnection.java:1585)
	at java.base/java.net.HttpURLConnection.getResponseCode(HttpURLConnection.java:529)
	at software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient$RequestCallable.getResponseCodeSafely(UrlConnectionHttpClient.java:419)
	at software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient$RequestCallable.call(UrlConnectionHttpClient.java:302)
	at software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient$RequestCallable.call(UrlConnectionHttpClient.java:269)
	at software.amazon.awssdk.core.internal.util.MetricUtils.measureDurationUnsafe(MetricUtils.java:63)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.MakeHttpRequestStage.executeHttpRequest(MakeHttpRequestStage.java:77)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.MakeHttpRequestStage.execute(MakeHttpRequestStage.java:56)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.MakeHttpRequestStage.execute(MakeHttpRequestStage.java:39)
	at software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206)
	at software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206)
	at software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206)
	at software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallAttemptTimeoutTrackingStage.execute(ApiCallAttemptTimeoutTrackingStage.java:73)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallAttemptTimeoutTrackingStage.execute(ApiCallAttemptTimeoutTrackingStage.java:42)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.TimeoutExceptionHandlingStage.execute(TimeoutExceptionHandlingStage.java:78)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.TimeoutExceptionHandlingStage.execute(TimeoutExceptionHandlingStage.java:40)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallAttemptMetricCollectionStage.execute(ApiCallAttemptMetricCollectionStage.java:50)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.ApiCallAttemptMetricCollectionStage.execute(ApiCallAttemptMetricCollectionStage.java:36)
	at software.amazon.awssdk.core.internal.http.pipeline.stages.RetryableStage.execute(RetryableStage.java:81)
	... 73 more


FAILED:  org.apache.solr.s3.S3BackupRepositoryTest.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.s3.S3BackupRepositoryTest: 
   1) Thread[id=64, name=qtp189515719-64, state=BLOCKED, group=TGRP-S3BackupRepositoryTest]
        at com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:133)
        at com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
        at java.base/java.lang.reflect.Method.invoke(Method.java:578)
        at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.Server.handle(Server.java:563)
        at org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base/java.lang.Thread.run(Thread.java:1589)
   2) Thread[id=58, name=qtp189515719-58, state=RUNNABLE, group=TGRP-S3BackupRepositoryTest]
        at java.base/sun.nio.ch.FileDispatcherImpl.read0(Native Method)
        at java.base/sun.nio.ch.FileDispatcherImpl.read(FileDispatcherImpl.java:48)
        at java.base/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:339)
        at java.base/sun.nio.ch.IOUtil.read(IOUtil.java:305)
        at java.base/sun.nio.ch.IOUtil.read(IOUtil.java:282)
        at java.base/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:240)
        at java.base/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:74)
        at java.base/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:103)
        at java.base/java.io.InputStream.read(InputStream.java:217)
        at java.base/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:82)
        at com.adobe.testing.s3mock.util.AwsChunkedDecodingInputStream.read(AwsChunkedDecodingInputStream.java:91)
        at java.base/java.io.InputStream.read(InputStream.java:293)
        at java.base/java.io.InputStream.read(InputStream.java:217)
        at com.adobe.testing.s3mock.store.ObjectStore.inputStreamToFile(ObjectStore.java:370)
        at com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:135)
        at com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
        at java.base/java.lang.reflect.Method.invoke(Method.java:578)
        at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.Server.handle(Server.java:563)
        at org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base/java.lang.Thread.run(Thread.java:1589)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.s3.S3BackupRepositoryTest: 
   1) Thread[id=64, name=qtp189515719-64, state=BLOCKED, group=TGRP-S3BackupRepositoryTest]
        at com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:133)
        at com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
        at java.base/java.lang.reflect.Method.invoke(Method.java:578)
        at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.Server.handle(Server.java:563)
        at org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base/java.lang.Thread.run(Thread.java:1589)
   2) Thread[id=58, name=qtp189515719-58, state=RUNNABLE, group=TGRP-S3BackupRepositoryTest]
        at java.base/sun.nio.ch.FileDispatcherImpl.read0(Native Method)
        at java.base/sun.nio.ch.FileDispatcherImpl.read(FileDispatcherImpl.java:48)
        at java.base/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:339)
        at java.base/sun.nio.ch.IOUtil.read(IOUtil.java:305)
        at java.base/sun.nio.ch.IOUtil.read(IOUtil.java:282)
        at java.base/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:240)
        at java.base/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:74)
        at java.base/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:103)
        at java.base/java.io.InputStream.read(InputStream.java:217)
        at java.base/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:82)
        at com.adobe.testing.s3mock.util.AwsChunkedDecodingInputStream.read(AwsChunkedDecodingInputStream.java:91)
        at java.base/java.io.InputStream.read(InputStream.java:293)
        at java.base/java.io.InputStream.read(InputStream.java:217)
        at com.adobe.testing.s3mock.store.ObjectStore.inputStreamToFile(ObjectStore.java:370)
        at com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:135)
        at com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
        at java.base/java.lang.reflect.Method.invoke(Method.java:578)
        at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.Server.handle(Server.java:563)
        at org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base/java.lang.Thread.run(Thread.java:1589)
	at __randomizedtesting.SeedInfo.seed([8DF81B0A5B0B9E2F]:0)


FAILED:  org.apache.solr.s3.S3BackupRepositoryTest.classMethod

Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=64, name=qtp189515719-64, state=BLOCKED, group=TGRP-S3BackupRepositoryTest]
        at com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:133)
        at com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
        at java.base/java.lang.reflect.Method.invoke(Method.java:578)
        at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.Server.handle(Server.java:563)
        at org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base/java.lang.Thread.run(Thread.java:1589)
   2) Thread[id=58, name=qtp189515719-58, state=RUNNABLE, group=TGRP-S3BackupRepositoryTest]
        at java.base/sun.nio.ch.FileDispatcherImpl.read0(Native Method)
        at java.base/sun.nio.ch.FileDispatcherImpl.read(FileDispatcherImpl.java:48)
        at java.base/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:339)
        at java.base/sun.nio.ch.IOUtil.read(IOUtil.java:305)
        at java.base/sun.nio.ch.IOUtil.read(IOUtil.java:282)
        at java.base/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:240)
        at java.base/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:74)
        at java.base/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:103)
        at java.base/java.io.InputStream.read(InputStream.java:217)
        at java.base/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:82)
        at com.adobe.testing.s3mock.util.AwsChunkedDecodingInputStream.read(AwsChunkedDecodingInputStream.java:91)
        at java.base/java.io.InputStream.read(InputStream.java:293)
        at java.base/java.io.InputStream.read(InputStream.java:217)
        at com.adobe.testing.s3mock.store.ObjectStore.inputStreamToFile(ObjectStore.java:370)
        at com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:135)
        at com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
        at java.base/java.lang.reflect.Method.invoke(Method.java:578)
        at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.Server.handle(Server.java:563)
        at org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base/java.lang.Thread.run(Thread.java:1589)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=64, name=qtp189515719-64, state=BLOCKED, group=TGRP-S3BackupRepositoryTest]
        at com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:133)
        at com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
        at java.base/java.lang.reflect.Method.invoke(Method.java:578)
        at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.Server.handle(Server.java:563)
        at org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base/java.lang.Thread.run(Thread.java:1589)
   2) Thread[id=58, name=qtp189515719-58, state=RUNNABLE, group=TGRP-S3BackupRepositoryTest]
        at java.base/sun.nio.ch.FileDispatcherImpl.read0(Native Method)
        at java.base/sun.nio.ch.FileDispatcherImpl.read(FileDispatcherImpl.java:48)
        at java.base/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:339)
        at java.base/sun.nio.ch.IOUtil.read(IOUtil.java:305)
        at java.base/sun.nio.ch.IOUtil.read(IOUtil.java:282)
        at java.base/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:240)
        at java.base/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:74)
        at java.base/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:103)
        at java.base/java.io.InputStream.read(InputStream.java:217)
        at java.base/sun.nio.ch.ChannelInputStream.read(ChannelInputStream.java:82)
        at com.adobe.testing.s3mock.util.AwsChunkedDecodingInputStream.read(AwsChunkedDecodingInputStream.java:91)
        at java.base/java.io.InputStream.read(InputStream.java:293)
        at java.base/java.io.InputStream.read(InputStream.java:217)
        at com.adobe.testing.s3mock.store.ObjectStore.inputStreamToFile(ObjectStore.java:370)
        at com.adobe.testing.s3mock.store.ObjectStore.storeS3ObjectMetadata(ObjectStore.java:135)
        at com.adobe.testing.s3mock.service.ObjectService.putS3Object(ObjectService.java:151)
        at com.adobe.testing.s3mock.ObjectController.putObject(ObjectController.java:541)
        at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
        at java.base/java.lang.reflect.Method.invoke(Method.java:578)
        at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
        at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
        at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
        at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
        at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
        at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1072)
        at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:965)
        at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
        at org.springframework.web.servlet.FrameworkServlet.doPut(FrameworkServlet.java:920)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:526)
        at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:590)
        at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764)
        at org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665)
        at com.adobe.testing.s3mock.KmsValidationFilter.doFilterInternal(KmsValidationFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:102)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:131)
        at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:578)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:223)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1385)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1307)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122)
        at org.eclipse.jetty.server.Server.handle(Server.java:563)
        at org.eclipse.jetty.server.HttpChannel.lambda$handle$0(HttpChannel.java:505)
        at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:762)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:497)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282)
        at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100)
        at org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:936)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1080)
        at java.base/java.lang.Thread.run(Thread.java:1589)
	at __randomizedtesting.SeedInfo.seed([8DF81B0A5B0B9E2F]:0)

[JENKINS] Solr-main-MacOSX (64bit/hotspot/jdk-16.0.2) - Build # 2597 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/2597/
Java: 64bit/hotspot/jdk-16.0.2 -XX:+UseCompressedOops -XX:+UseG1GC

1 tests failed.
FAILED:  org.apache.solr.cloud.ReindexCollectionTest.testAbort

Error Message:
java.lang.AssertionError: {responseHeader={status=0, QTime=12}, status={state=running, msg=found [08fd0fff-de6b-4979-8091-aa2d2f437cf3] in running tasks}}

Stack Trace:
java.lang.AssertionError: {responseHeader={status=0, QTime=12}, status={state=running, msg=found [08fd0fff-de6b-4979-8091-aa2d2f437cf3] in running tasks}}
	at __randomizedtesting.SeedInfo.seed([B80C2DD3568C5D99:DE6F9A442256CFE0]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.junit.Assert.assertTrue(Assert.java:42)
	at org.junit.Assert.assertNotNull(Assert.java:713)
	at org.apache.solr.cloud.ReindexCollectionTest.testAbort(ReindexCollectionTest.java:425)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:78)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:567)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
	at java.base/java.lang.Thread.run(Thread.java:831)

[JENKINS] Solr-main-MacOSX (64bit/hotspot/jdk-18) - Build # 2596 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/2596/
Java: 64bit/hotspot/jdk-18 -XX:+UseCompressedOops -XX:+UseParallelGC

1 tests failed.
FAILED:  org.apache.solr.search.TestCoordinatorRole.testNRTRestart

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occurred when talking to server at: http://127.0.0.1:59079/solr

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occurred when talking to server at: http://127.0.0.1:59079/solr
	at __randomizedtesting.SeedInfo.seed([F717E3090B0B2BAC:78353C2A916F97A]:0)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:731)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:234)
	at app//org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:466)
	at app//org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:423)
	at app//org.apache.solr.search.TestCoordinatorRole.testNRTRestart(TestCoordinatorRole.java:240)
	at java.base@18/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
	at java.base@18/java.lang.reflect.Method.invoke(Method.java:577)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
	at java.base@18/java.lang.Thread.run(Thread.java:833)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:59079 failed to respond
	at app//org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:141)
	at app//org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
	at app//org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
	at app//org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
	at app//org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:157)
	at app//org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
	at app//org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
	at app//org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
	at app//org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186)
	at app//org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
	at app//org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at app//org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
	at app//org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
	at app//org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:576)
	... 51 more

[JENKINS] Solr-main-MacOSX (64bit/hotspot/jdk-17.0.5) - Build # 2595 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Solr-main-MacOSX/2595/
Java: 64bit/hotspot/jdk-17.0.5 -XX:-UseCompressedOops -XX:+UseG1GC

4 tests failed.
FAILED:  org.apache.solr.cloud.TestPrepRecovery.testLeaderNotResponding

Error Message:
java.lang.AssertionError: Expected collection: testLeaderNotResponding to be live with 1 shard and 2 replicas
Timeout waiting to see state for collection=testLeaderNotResponding :DocCollection(testLeaderNotResponding//collections/testLeaderNotResponding/state.json/7 )={
  "pullReplicas":"0",
  "configName":"testLeaderNotResponding.AUTOCREATED",
  "replicationFactor":"1",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node2":{
          "core":"testLeaderNotResponding_shard1_replica_n1",
          "leader":"true",
          "node_name":"127.0.0.1:60315_solr",
          "base_url":"http://127.0.0.1:60315/solr",
          "state":"active",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node4":{
          "core":"testLeaderNotResponding_shard1_replica_n3",
          "node_name":"127.0.0.1:60316_solr",
          "base_url":"http://127.0.0.1:60316/solr",
          "state":"recovering",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"1",
  "tlogReplicas":"0"}
Live Nodes: [127.0.0.1:60315_solr, 127.0.0.1:60316_solr, 127.0.0.1:60394_solr]
Last available state: DocCollection(testLeaderNotResponding//collections/testLeaderNotResponding/state.json/7 )={
  "pullReplicas":"0",
  "configName":"testLeaderNotResponding.AUTOCREATED",
  "replicationFactor":"1",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node2":{
          "core":"testLeaderNotResponding_shard1_replica_n1",
          "leader":"true",
          "node_name":"127.0.0.1:60315_solr",
          "base_url":"http://127.0.0.1:60315/solr",
          "state":"active",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node4":{
          "core":"testLeaderNotResponding_shard1_replica_n3",
          "node_name":"127.0.0.1:60316_solr",
          "base_url":"http://127.0.0.1:60316/solr",
          "state":"recovering",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"1",
  "tlogReplicas":"0"}

Stack Trace:
java.lang.AssertionError: Expected collection: testLeaderNotResponding to be live with 1 shard and 2 replicas
Timeout waiting to see state for collection=testLeaderNotResponding :DocCollection(testLeaderNotResponding//collections/testLeaderNotResponding/state.json/7 )={
  "pullReplicas":"0",
  "configName":"testLeaderNotResponding.AUTOCREATED",
  "replicationFactor":"1",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node2":{
          "core":"testLeaderNotResponding_shard1_replica_n1",
          "leader":"true",
          "node_name":"127.0.0.1:60315_solr",
          "base_url":"http://127.0.0.1:60315/solr",
          "state":"active",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node4":{
          "core":"testLeaderNotResponding_shard1_replica_n3",
          "node_name":"127.0.0.1:60316_solr",
          "base_url":"http://127.0.0.1:60316/solr",
          "state":"recovering",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"1",
  "tlogReplicas":"0"}
Live Nodes: [127.0.0.1:60315_solr, 127.0.0.1:60316_solr, 127.0.0.1:60394_solr]
Last available state: DocCollection(testLeaderNotResponding//collections/testLeaderNotResponding/state.json/7 )={
  "pullReplicas":"0",
  "configName":"testLeaderNotResponding.AUTOCREATED",
  "replicationFactor":"1",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node2":{
          "core":"testLeaderNotResponding_shard1_replica_n1",
          "leader":"true",
          "node_name":"127.0.0.1:60315_solr",
          "base_url":"http://127.0.0.1:60315/solr",
          "state":"active",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node4":{
          "core":"testLeaderNotResponding_shard1_replica_n3",
          "node_name":"127.0.0.1:60316_solr",
          "base_url":"http://127.0.0.1:60316/solr",
          "state":"recovering",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"1",
  "tlogReplicas":"0"}
	at __randomizedtesting.SeedInfo.seed([698C73BB0588A709:30D05BDC42FF67B6]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.solr.cloud.SolrCloudTestCase.waitForState(SolrCloudTestCase.java:171)
	at org.apache.solr.cloud.TestPrepRecovery.testLeaderNotResponding(TestPrepRecovery.java:110)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:568)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
	at java.base/java.lang.Thread.run(Thread.java:833)


FAILED:  org.apache.solr.cloud.UnloadDistributedZkTest.test

Error Message:
org.apache.solr.common.SolrException: No registered leader was found after waiting for 15000ms , collection: unloadcollection slice: shard1 saw state=DocCollection(unloadcollection//collections/unloadcollection/state.json/14 )={
  "pullReplicas":"0",
  "configName":"conf1",
  "replicationFactor":"1",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node3":{
          "core":"unloadcollection_shard1_replica2",
          "leader":"true",
          "node_name":"127.0.0.1:61365_a_d%2Fj",
          "base_url":"http://127.0.0.1:61365/a_d/j",
          "state":"active",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node4":{
          "core":"unloadcollection_shard1_replica3",
          "node_name":"127.0.0.1:61384_a_d%2Fj",
          "base_url":"http://127.0.0.1:61384/a_d/j",
          "state":"active",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"1",
  "tlogReplicas":"0"} with live_nodes=[127.0.0.1:61342_a_d%2Fj, 127.0.0.1:61384_a_d%2Fj, 127.0.0.1:61304_a_d%2Fj, 127.0.0.1:61365_a_d%2Fj, 127.0.0.1:61400_a_d%2Fj]

Stack Trace:
org.apache.solr.common.SolrException: No registered leader was found after waiting for 15000ms , collection: unloadcollection slice: shard1 saw state=DocCollection(unloadcollection//collections/unloadcollection/state.json/14 )={
  "pullReplicas":"0",
  "configName":"conf1",
  "replicationFactor":"1",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node3":{
          "core":"unloadcollection_shard1_replica2",
          "leader":"true",
          "node_name":"127.0.0.1:61365_a_d%2Fj",
          "base_url":"http://127.0.0.1:61365/a_d/j",
          "state":"active",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node4":{
          "core":"unloadcollection_shard1_replica3",
          "node_name":"127.0.0.1:61384_a_d%2Fj",
          "base_url":"http://127.0.0.1:61384/a_d/j",
          "state":"active",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"1",
  "tlogReplicas":"0"} with live_nodes=[127.0.0.1:61342_a_d%2Fj, 127.0.0.1:61384_a_d%2Fj, 127.0.0.1:61304_a_d%2Fj, 127.0.0.1:61365_a_d%2Fj, 127.0.0.1:61400_a_d%2Fj]
	at __randomizedtesting.SeedInfo.seed([698C73BB0588A709:E1D84C61AB74CAF1]:0)
	at app//org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:970)
	at app//org.apache.solr.common.cloud.ZkStateReader.getLeaderUrl(ZkStateReader.java:900)
	at app//org.apache.solr.cloud.AbstractUnloadDistributedZkTestBase.testCoreUnloadAndLeaders(AbstractUnloadDistributedZkTestBase.java:311)
	at app//org.apache.solr.cloud.AbstractUnloadDistributedZkTestBase.test(AbstractUnloadDistributedZkTestBase.java:80)
	at java.base@17.0.5/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base@17.0.5/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base@17.0.5/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base@17.0.5/java.lang.reflect.Method.invoke(Method.java:568)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1164)
	at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1135)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
	at java.base@17.0.5/java.lang.Thread.run(Thread.java:833)


FAILED:  org.apache.solr.cloud.api.collections.CollectionsAPIDistributedZkTest.testCoresAreDistributedAcrossNodes

Error Message:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:61870/solr: Underlying core creation failed while creating collection: nodes_used_collection

Stack Trace:
org.apache.solr.client.solrj.impl.BaseHttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:61870/solr: Underlying core creation failed while creating collection: nodes_used_collection
	at __randomizedtesting.SeedInfo.seed([698C73BB0588A709:DCCA6D7BE462F18]:0)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:719)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:386)
	at app//org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:352)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1133)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at app//org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:767)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:234)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:249)
	at app//org.apache.solr.cloud.api.collections.AbstractCollectionsAPIDistributedZkTestBase.testCoresAreDistributedAcrossNodes(AbstractCollectionsAPIDistributedZkTestBase.java:320)
	at java.base@17.0.5/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base@17.0.5/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base@17.0.5/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base@17.0.5/java.lang.reflect.Method.invoke(Method.java:568)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
	at java.base@17.0.5/java.lang.Thread.run(Thread.java:833)


FAILED:  org.apache.solr.search.TestCoordinatorRole.testNRTRestart

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occurred when talking to server at: http://127.0.0.1:50924/solr

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occurred when talking to server at: http://127.0.0.1:50924/solr
	at __randomizedtesting.SeedInfo.seed([698C73BB0588A709:9918C370A79575DF]:0)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:731)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:234)
	at app//org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:466)
	at app//org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:423)
	at app//org.apache.solr.search.TestCoordinatorRole.testNRTRestart(TestCoordinatorRole.java:240)
	at java.base@17.0.5/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base@17.0.5/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base@17.0.5/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base@17.0.5/java.lang.reflect.Method.invoke(Method.java:568)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:44)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
	at app//org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
	at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
	at app//org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
	at app//org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
	at app//org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
	at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
	at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
	at java.base@17.0.5/java.lang.Thread.run(Thread.java:833)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:50924 failed to respond
	at app//org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:141)
	at app//org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
	at app//org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
	at app//org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
	at app//org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:157)
	at app//org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
	at app//org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
	at app//org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
	at app//org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186)
	at app//org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
	at app//org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at app//org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
	at app//org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
	at app//org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
	at app//org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:576)
	... 53 more