You are viewing a plain text version of this content. The canonical link for it is here.
Posted to builds@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2021/06/16 16:22:54 UTC
[JENKINS-EA] Lucene-Solr-8.9-Linux (64bit/jdk-17-ea+25) - Build #
195 - Still Unstable!
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-8.9-Linux/195/
Java: 64bit/jdk-17-ea+25 -XX:+UseCompressedOops -XX:+UseG1GC
1 tests failed.
FAILED: org.apache.solr.common.cloud.PerReplicaStatesIntegrationTest.testPerReplicaStateCollection
Error Message:
No live SolrServers available to handle this request:[http://127.0.0.1:37875/solr]
Stack Trace:
org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[http://127.0.0.1:37875/solr]
at __randomizedtesting.SeedInfo.seed([DDB288C5F894ED1E:F78F238758343713]:0)
at org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:346)
at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.sendRequest(BaseCloudSolrClient.java:1171)
at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.requestWithRetryOnStaleState(BaseCloudSolrClient.java:934)
at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.request(BaseCloudSolrClient.java:866)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:214)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:231)
at org.apache.solr.common.cloud.PerReplicaStatesIntegrationTest.testPerReplicaStateCollection(PerReplicaStatesIntegrationTest.java:84)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:568)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.base/java.lang.Thread.run(Thread.java:833)
Caused by: org.apache.solr.client.solrj.SolrServerException: Timeout occurred while waiting response from server at: http://127.0.0.1:37875/solr
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:692)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:266)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
at org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:369)
at org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:297)
... 48 more
Caused by: java.net.SocketTimeoutException: Read timed out
at java.base/sun.nio.ch.NioSocketImpl.timedRead(NioSocketImpl.java:283)
at java.base/sun.nio.ch.NioSocketImpl.implRead(NioSocketImpl.java:309)
at java.base/sun.nio.ch.NioSocketImpl.read(NioSocketImpl.java:350)
at java.base/sun.nio.ch.NioSocketImpl$1.read(NioSocketImpl.java:803)
at java.base/java.net.Socket$SocketInputStream.read(Socket.java:966)
at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:280)
at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:157)
at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186)
at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:571)
... 52 more
[JENKINS] Lucene-Solr-8.9-Linux (64bit/jdk1.8.0_201) - Build # 197
- Still Unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-8.9-Linux/197/
Java: 64bit/jdk1.8.0_201 -XX:+UseCompressedOops -XX:+UseConcMarkSweepGC
3 tests failed.
FAILED: org.apache.solr.cloud.LeaderTragicEventTest.testLeaderFailsOver
Error Message:
Error from server at https://127.0.0.1:39455/solr: Underlying core creation failed while creating collection: testLeaderFailsOver
Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:39455/solr: Underlying core creation failed while creating collection: testLeaderFailsOver
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:681)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:266)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
at org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:369)
at org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:297)
at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.sendRequest(BaseCloudSolrClient.java:1171)
at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.requestWithRetryOnStaleState(BaseCloudSolrClient.java:934)
at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.request(BaseCloudSolrClient.java:866)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:214)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:231)
at org.apache.solr.cloud.LeaderTragicEventTest.testLeaderFailsOver(LeaderTragicEventTest.java:80)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.LeaderTragicEventTest.testLeaderFailsOver
Error Message:
Error from server at https://127.0.0.1:40099/solr: Underlying core creation failed while creating collection: testLeaderFailsOver
Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:40099/solr: Underlying core creation failed while creating collection: testLeaderFailsOver
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:681)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:266)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
at org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:369)
at org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:297)
at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.sendRequest(BaseCloudSolrClient.java:1171)
at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.requestWithRetryOnStaleState(BaseCloudSolrClient.java:934)
at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.request(BaseCloudSolrClient.java:866)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:214)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:231)
at org.apache.solr.cloud.LeaderTragicEventTest.testLeaderFailsOver(LeaderTragicEventTest.java:80)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.handler.component.TestDistributedStatsComponentCardinality.test
Error Message:
.responseHeader.status:200!=0
Stack Trace:
junit.framework.AssertionFailedError: .responseHeader.status:200!=0
at __randomizedtesting.SeedInfo.seed([A5E36629D32F89FD:2DB759F37DD3E405]:0)
at junit.framework.Assert.fail(Assert.java:57)
at org.apache.solr.BaseDistributedSearchTestCase.compareSolrResponses(BaseDistributedSearchTestCase.java:1006)
at org.apache.solr.BaseDistributedSearchTestCase.compareResponses(BaseDistributedSearchTestCase.java:1033)
at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:679)
at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:642)
at org.apache.solr.handler.component.TestDistributedStatsComponentCardinality.test(TestDistributedStatsComponentCardinality.java:142)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1090)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1061)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
[JENKINS] Lucene-Solr-8.9-Linux (64bit/jdk1.8.0_201) - Build # 197
- Still Unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-8.9-Linux/197/
Java: 64bit/jdk1.8.0_201 -XX:+UseCompressedOops -XX:+UseConcMarkSweepGC
3 tests failed.
FAILED: org.apache.solr.cloud.LeaderTragicEventTest.testLeaderFailsOver
Error Message:
Error from server at https://127.0.0.1:39455/solr: Underlying core creation failed while creating collection: testLeaderFailsOver
Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:39455/solr: Underlying core creation failed while creating collection: testLeaderFailsOver
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:681)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:266)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
at org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:369)
at org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:297)
at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.sendRequest(BaseCloudSolrClient.java:1171)
at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.requestWithRetryOnStaleState(BaseCloudSolrClient.java:934)
at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.request(BaseCloudSolrClient.java:866)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:214)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:231)
at org.apache.solr.cloud.LeaderTragicEventTest.testLeaderFailsOver(LeaderTragicEventTest.java:80)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.LeaderTragicEventTest.testLeaderFailsOver
Error Message:
Error from server at https://127.0.0.1:40099/solr: Underlying core creation failed while creating collection: testLeaderFailsOver
Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:40099/solr: Underlying core creation failed while creating collection: testLeaderFailsOver
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:681)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:266)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
at org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:369)
at org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:297)
at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.sendRequest(BaseCloudSolrClient.java:1171)
at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.requestWithRetryOnStaleState(BaseCloudSolrClient.java:934)
at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.request(BaseCloudSolrClient.java:866)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:214)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:231)
at org.apache.solr.cloud.LeaderTragicEventTest.testLeaderFailsOver(LeaderTragicEventTest.java:80)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.handler.component.TestDistributedStatsComponentCardinality.test
Error Message:
.responseHeader.status:200!=0
Stack Trace:
junit.framework.AssertionFailedError: .responseHeader.status:200!=0
at __randomizedtesting.SeedInfo.seed([A5E36629D32F89FD:2DB759F37DD3E405]:0)
at junit.framework.Assert.fail(Assert.java:57)
at org.apache.solr.BaseDistributedSearchTestCase.compareSolrResponses(BaseDistributedSearchTestCase.java:1006)
at org.apache.solr.BaseDistributedSearchTestCase.compareResponses(BaseDistributedSearchTestCase.java:1033)
at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:679)
at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:642)
at org.apache.solr.handler.component.TestDistributedStatsComponentCardinality.test(TestDistributedStatsComponentCardinality.java:142)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1090)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1061)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
[JENKINS] Lucene-Solr-8.9-Linux (64bit/jdk-16) - Build # 196 -
Still Unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-8.9-Linux/196/
Java: 64bit/jdk-16 -XX:+UseCompressedOops -XX:+UseG1GC
4 tests failed.
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest
Error Message:
55 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest: 1) Thread[id=9460, name=qtp1312166695-9460-acceptor-0@486ac7a4-ServerConnector@401f7e19{ssl, (ssl, alpn, http/1.1, h2)}{127.0.0.1:40869}, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:714) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:937) at java.base@16/java.util.concurrent.locks.ReentrantLock$Sync.lock(ReentrantLock.java:153) at java.base@16/java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock.java:322) at java.base@16/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:385) at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388) at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:702) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 2) Thread[id=9484, name=qtp153449517-9484, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 3) Thread[id=9571, name=SolrRrdBackendFactory-6609-thread-2, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505) at java.base@16/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at java.base@16/java.lang.Thread.run(Thread.java:831) 4) Thread[id=9451, name=qtp1312166695-9451, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 5) Thread[id=9563, name=qtp1312166695-9563, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023) at java.base@16/java.lang.Thread.run(Thread.java:831) 6) Thread[id=9565, name=Connector-Scheduler-5bbc5236-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at java.base@16/java.lang.Thread.run(Thread.java:831) 7) Thread[id=9454, name=qtp1312166695-9454, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 8) Thread[id=9532, name=MetricsHistoryHandler-6610-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/java.lang.Thread.sleep(Native Method) at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:138) at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:89) at app//org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:351) at app//org.apache.solr.client.solrj.impl.ZkDistribStateManager.getData(ZkDistribStateManager.java:88) at app//org.apache.solr.client.solrj.cloud.DistribStateManager.getData(DistribStateManager.java:56) at app//org.apache.solr.handler.admin.MetricsHistoryHandler.getOverseerLeader(MetricsHistoryHandler.java:335) at app//org.apache.solr.handler.admin.MetricsHistoryHandler.amIOverseerLeader(MetricsHistoryHandler.java:373) at app//org.apache.solr.handler.admin.MetricsHistoryHandler.amIOverseerLeader(MetricsHistoryHandler.java:368) at app//org.apache.solr.handler.admin.MetricsHistoryHandler.collectGlobalMetrics(MetricsHistoryHandler.java:469) at app//org.apache.solr.handler.admin.MetricsHistoryHandler.collectMetrics(MetricsHistoryHandler.java:395) at app//org.apache.solr.handler.admin.MetricsHistoryHandler.lambda$new$0(MetricsHistoryHandler.java:242) at app//org.apache.solr.handler.admin.MetricsHistoryHandler$$Lambda$954/0x0000000801361e60.run(Unknown Source) at java.base@16/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base@16/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at java.base@16/java.lang.Thread.run(Thread.java:831) 9) Thread[id=9572, name=SolrRrdBackendFactory-6619-thread-2, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505) at java.base@16/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at java.base@16/java.lang.Thread.run(Thread.java:831) 10) Thread[id=9452, name=qtp1312166695-9452, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 11) Thread[id=9554, name=Connector-Scheduler-401f7e19-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at java.base@16/java.lang.Thread.run(Thread.java:831) 12) Thread[id=9531, name=SolrRrdBackendFactory-6609-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at java.base@16/java.lang.Thread.run(Thread.java:831) 13) Thread[id=9673, name=qtp153449517-9673, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023) at java.base@16/java.lang.Thread.run(Thread.java:831) 14) Thread[id=9483, name=qtp153449517-9483, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 15) Thread[id=9683, name=qtp1312166695-9683, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023) at java.base@16/java.lang.Thread.run(Thread.java:831) 16) Thread[id=9542, name=MetricsHistoryHandler-6620-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/java.lang.Thread.sleep(Native Method) at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:138) at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:89) at app//org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:351) at app//org.apache.solr.client.solrj.impl.ZkDistribStateManager.getData(ZkDistribStateManager.java:88) at app//org.apache.solr.client.solrj.cloud.DistribStateManager.getData(DistribStateManager.java:56) at app//org.apache.solr.handler.admin.MetricsHistoryHandler.getOverseerLeader(MetricsHistoryHandler.java:335) at app//org.apache.solr.handler.admin.MetricsHistoryHandler.amIOverseerLeader(MetricsHistoryHandler.java:373) at app//org.apache.solr.handler.admin.MetricsHistoryHandler.amIOverseerLeader(MetricsHistoryHandler.java:368) at app//org.apache.solr.handler.admin.MetricsHistoryHandler.collectGlobalMetrics(MetricsHistoryHandler.java:469) at app//org.apache.solr.handler.admin.MetricsHistoryHandler.collectMetrics(MetricsHistoryHandler.java:395) at app//org.apache.solr.handler.admin.MetricsHistoryHandler.lambda$new$0(MetricsHistoryHandler.java:242) at app//org.apache.solr.handler.admin.MetricsHistoryHandler$$Lambda$954/0x0000000801361e60.run(Unknown Source) at java.base@16/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base@16/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at java.base@16/java.lang.Thread.run(Thread.java:831) 17) Thread[id=9459, name=Session-HouseKeeper-ff43509-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at java.base@16/java.lang.Thread.run(Thread.java:831) 18) Thread[id=9682, name=qtp1312166695-9682, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023) at java.base@16/java.lang.Thread.run(Thread.java:831) 19) Thread[id=9487, name=qtp153449517-9487, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 20) Thread[id=9485, name=qtp153449517-9485, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 21) Thread[id=9476, name=qtp1931265301-9476-acceptor-1@11a1a8e1-ServerConnector@16f86070{ssl, (ssl, alpn, http/1.1, h2)}{127.0.0.1:45021}, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.Net.accept(Native Method) at java.base@16/sun.nio.ch.ServerSocketChannelImpl.implAccept(ServerSocketChannelImpl.java:424) at java.base@16/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:390) at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388) at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:702) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 22) Thread[id=9471, name=qtp1931265301-9471, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 23) Thread[id=9569, name=qtp1931265301-9569, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023) at java.base@16/java.lang.Thread.run(Thread.java:831) 24) Thread[id=9468, name=qtp1931265301-9468, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 25) Thread[id=9568, name=Connector-Scheduler-16f86070-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at java.base@16/java.lang.Thread.run(Thread.java:831) 26) Thread[id=9466, name=qtp1931265301-9466, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 27) Thread[id=9473, name=qtp1931265301-9473, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 28) Thread[id=9474, name=Session-HouseKeeper-2d64b24c-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at java.base@16/java.lang.Thread.run(Thread.java:831) 29) Thread[id=9475, name=qtp1931265301-9475-acceptor-0@64e0dff8-ServerConnector@16f86070{ssl, (ssl, alpn, http/1.1, h2)}{127.0.0.1:45021}, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:714) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:937) at java.base@16/java.util.concurrent.locks.ReentrantLock$Sync.lock(ReentrantLock.java:153) at java.base@16/java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock.java:322) at java.base@16/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:385) at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388) at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:702) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 30) Thread[id=9482, name=qtp153449517-9482, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 31) Thread[id=9570, name=SolrRrdBackendFactory-6588-thread-2, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505) at java.base@16/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at java.base@16/java.lang.Thread.run(Thread.java:831) 32) Thread[id=9541, name=SolrRrdBackendFactory-6619-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at java.base@16/java.lang.Thread.run(Thread.java:831) 33) Thread[id=9533, name=Thread-1811, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/java.lang.Object.wait(Native Method) at java.base@16/java.lang.Object.wait(Object.java:320) at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2281) 34) Thread[id=9469, name=qtp1931265301-9469, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 35) Thread[id=9486, name=qtp153449517-9486, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 36) Thread[id=9453, name=qtp1312166695-9453, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 37) Thread[id=9491, name=qtp153449517-9491-acceptor-1@7dd30796-ServerConnector@5bbc5236{ssl, (ssl, alpn, http/1.1, h2)}{127.0.0.1:46045}, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:714) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:937) at java.base@16/java.util.concurrent.locks.ReentrantLock$Sync.lock(ReentrantLock.java:153) at java.base@16/java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock.java:322) at java.base@16/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:385) at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388) at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:702) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 38) Thread[id=9470, name=qtp1931265301-9470, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 39) Thread[id=9467, name=qtp1931265301-9467, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 40) Thread[id=9515, name=MetricsHistoryHandler-6589-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/java.lang.Thread.sleep(Native Method) at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:138) at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:89) at app//org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:351) at app//org.apache.solr.client.solrj.impl.ZkDistribStateManager.getData(ZkDistribStateManager.java:88) at app//org.apache.solr.client.solrj.cloud.DistribStateManager.getData(DistribStateManager.java:56) at app//org.apache.solr.handler.admin.MetricsHistoryHandler.getOverseerLeader(MetricsHistoryHandler.java:335) at app//org.apache.solr.handler.admin.MetricsHistoryHandler.amIOverseerLeader(MetricsHistoryHandler.java:373) at app//org.apache.solr.handler.admin.MetricsHistoryHandler.amIOverseerLeader(MetricsHistoryHandler.java:368) at app//org.apache.solr.handler.admin.MetricsHistoryHandler.collectGlobalMetrics(MetricsHistoryHandler.java:469) at app//org.apache.solr.handler.admin.MetricsHistoryHandler.collectMetrics(MetricsHistoryHandler.java:395) at app//org.apache.solr.handler.admin.MetricsHistoryHandler.lambda$new$0(MetricsHistoryHandler.java:242) at app//org.apache.solr.handler.admin.MetricsHistoryHandler$$Lambda$954/0x0000000801361e60.run(Unknown Source) at java.base@16/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base@16/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at java.base@16/java.lang.Thread.run(Thread.java:831) 41) Thread[id=9516, name=Thread-1810, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/java.lang.Object.wait(Native Method) at java.base@16/java.lang.Object.wait(Object.java:320) at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2281) 42) Thread[id=9472, name=qtp1931265301-9472, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 43) Thread[id=9489, name=Session-HouseKeeper-63b1bc1d-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at java.base@16/java.lang.Thread.run(Thread.java:831) 44) Thread[id=9456, name=qtp1312166695-9456, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 45) Thread[id=9455, name=qtp1312166695-9455, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 46) Thread[id=9564, name=qtp153449517-9564, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023) at java.base@16/java.lang.Thread.run(Thread.java:831) 47) Thread[id=9458, name=qtp1312166695-9458, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 48) Thread[id=9490, name=qtp153449517-9490-acceptor-0@47b75565-ServerConnector@5bbc5236{ssl, (ssl, alpn, http/1.1, h2)}{127.0.0.1:46045}, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.Net.accept(Native Method) at java.base@16/sun.nio.ch.ServerSocketChannelImpl.implAccept(ServerSocketChannelImpl.java:424) at java.base@16/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:390) at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388) at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:702) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 49) Thread[id=9514, name=SolrRrdBackendFactory-6588-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at java.base@16/java.lang.Thread.run(Thread.java:831) 50) Thread[id=9675, name=qtp1931265301-9675, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023) at java.base@16/java.lang.Thread.run(Thread.java:831) 51) Thread[id=9543, name=Thread-1812, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/java.lang.Object.wait(Native Method) at java.base@16/java.lang.Object.wait(Object.java:320) at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2281) 52) Thread[id=9481, name=qtp153449517-9481, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 53) Thread[id=9488, name=qtp153449517-9488, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 54) Thread[id=9461, name=qtp1312166695-9461-acceptor-1@13d0161d-ServerConnector@401f7e19{ssl, (ssl, alpn, http/1.1, h2)}{127.0.0.1:40869}, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.Net.accept(Native Method) at java.base@16/sun.nio.ch.ServerSocketChannelImpl.implAccept(ServerSocketChannelImpl.java:424) at java.base@16/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:390) at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388) at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:702) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 55) Thread[id=9457, name=qtp1312166695-9457, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 55 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest:
1) Thread[id=9460, name=qtp1312166695-9460-acceptor-0@486ac7a4-ServerConnector@401f7e19{ssl, (ssl, alpn, http/1.1, h2)}{127.0.0.1:40869}, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:714)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:937)
at java.base@16/java.util.concurrent.locks.ReentrantLock$Sync.lock(ReentrantLock.java:153)
at java.base@16/java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock.java:322)
at java.base@16/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:385)
at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388)
at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:702)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
2) Thread[id=9484, name=qtp153449517-9484, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
3) Thread[id=9571, name=SolrRrdBackendFactory-6609-thread-2, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
at java.base@16/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
at java.base@16/java.lang.Thread.run(Thread.java:831)
4) Thread[id=9451, name=qtp1312166695-9451, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
5) Thread[id=9563, name=qtp1312166695-9563, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023)
at java.base@16/java.lang.Thread.run(Thread.java:831)
6) Thread[id=9565, name=Connector-Scheduler-5bbc5236-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
at java.base@16/java.lang.Thread.run(Thread.java:831)
7) Thread[id=9454, name=qtp1312166695-9454, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
8) Thread[id=9532, name=MetricsHistoryHandler-6610-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/java.lang.Thread.sleep(Native Method)
at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:138)
at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:89)
at app//org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:351)
at app//org.apache.solr.client.solrj.impl.ZkDistribStateManager.getData(ZkDistribStateManager.java:88)
at app//org.apache.solr.client.solrj.cloud.DistribStateManager.getData(DistribStateManager.java:56)
at app//org.apache.solr.handler.admin.MetricsHistoryHandler.getOverseerLeader(MetricsHistoryHandler.java:335)
at app//org.apache.solr.handler.admin.MetricsHistoryHandler.amIOverseerLeader(MetricsHistoryHandler.java:373)
at app//org.apache.solr.handler.admin.MetricsHistoryHandler.amIOverseerLeader(MetricsHistoryHandler.java:368)
at app//org.apache.solr.handler.admin.MetricsHistoryHandler.collectGlobalMetrics(MetricsHistoryHandler.java:469)
at app//org.apache.solr.handler.admin.MetricsHistoryHandler.collectMetrics(MetricsHistoryHandler.java:395)
at app//org.apache.solr.handler.admin.MetricsHistoryHandler.lambda$new$0(MetricsHistoryHandler.java:242)
at app//org.apache.solr.handler.admin.MetricsHistoryHandler$$Lambda$954/0x0000000801361e60.run(Unknown Source)
at java.base@16/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
at java.base@16/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
at java.base@16/java.lang.Thread.run(Thread.java:831)
9) Thread[id=9572, name=SolrRrdBackendFactory-6619-thread-2, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
at java.base@16/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
at java.base@16/java.lang.Thread.run(Thread.java:831)
10) Thread[id=9452, name=qtp1312166695-9452, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
11) Thread[id=9554, name=Connector-Scheduler-401f7e19-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
at java.base@16/java.lang.Thread.run(Thread.java:831)
12) Thread[id=9531, name=SolrRrdBackendFactory-6609-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
at java.base@16/java.lang.Thread.run(Thread.java:831)
13) Thread[id=9673, name=qtp153449517-9673, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023)
at java.base@16/java.lang.Thread.run(Thread.java:831)
14) Thread[id=9483, name=qtp153449517-9483, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
15) Thread[id=9683, name=qtp1312166695-9683, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023)
at java.base@16/java.lang.Thread.run(Thread.java:831)
16) Thread[id=9542, name=MetricsHistoryHandler-6620-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/java.lang.Thread.sleep(Native Method)
at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:138)
at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:89)
at app//org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:351)
at app//org.apache.solr.client.solrj.impl.ZkDistribStateManager.getData(ZkDistribStateManager.java:88)
at app//org.apache.solr.client.solrj.cloud.DistribStateManager.getData(DistribStateManager.java:56)
at app//org.apache.solr.handler.admin.MetricsHistoryHandler.getOverseerLeader(MetricsHistoryHandler.java:335)
at app//org.apache.solr.handler.admin.MetricsHistoryHandler.amIOverseerLeader(MetricsHistoryHandler.java:373)
at app//org.apache.solr.handler.admin.MetricsHistoryHandler.amIOverseerLeader(MetricsHistoryHandler.java:368)
at app//org.apache.solr.handler.admin.MetricsHistoryHandler.collectGlobalMetrics(MetricsHistoryHandler.java:469)
at app//org.apache.solr.handler.admin.MetricsHistoryHandler.collectMetrics(MetricsHistoryHandler.java:395)
at app//org.apache.solr.handler.admin.MetricsHistoryHandler.lambda$new$0(MetricsHistoryHandler.java:242)
at app//org.apache.solr.handler.admin.MetricsHistoryHandler$$Lambda$954/0x0000000801361e60.run(Unknown Source)
at java.base@16/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
at java.base@16/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
at java.base@16/java.lang.Thread.run(Thread.java:831)
17) Thread[id=9459, name=Session-HouseKeeper-ff43509-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
at java.base@16/java.lang.Thread.run(Thread.java:831)
18) Thread[id=9682, name=qtp1312166695-9682, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023)
at java.base@16/java.lang.Thread.run(Thread.java:831)
19) Thread[id=9487, name=qtp153449517-9487, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
20) Thread[id=9485, name=qtp153449517-9485, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
21) Thread[id=9476, name=qtp1931265301-9476-acceptor-1@11a1a8e1-ServerConnector@16f86070{ssl, (ssl, alpn, http/1.1, h2)}{127.0.0.1:45021}, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.Net.accept(Native Method)
at java.base@16/sun.nio.ch.ServerSocketChannelImpl.implAccept(ServerSocketChannelImpl.java:424)
at java.base@16/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:390)
at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388)
at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:702)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
22) Thread[id=9471, name=qtp1931265301-9471, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
23) Thread[id=9569, name=qtp1931265301-9569, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023)
at java.base@16/java.lang.Thread.run(Thread.java:831)
24) Thread[id=9468, name=qtp1931265301-9468, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
25) Thread[id=9568, name=Connector-Scheduler-16f86070-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
at java.base@16/java.lang.Thread.run(Thread.java:831)
26) Thread[id=9466, name=qtp1931265301-9466, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
27) Thread[id=9473, name=qtp1931265301-9473, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
28) Thread[id=9474, name=Session-HouseKeeper-2d64b24c-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
at java.base@16/java.lang.Thread.run(Thread.java:831)
29) Thread[id=9475, name=qtp1931265301-9475-acceptor-0@64e0dff8-ServerConnector@16f86070{ssl, (ssl, alpn, http/1.1, h2)}{127.0.0.1:45021}, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:714)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:937)
at java.base@16/java.util.concurrent.locks.ReentrantLock$Sync.lock(ReentrantLock.java:153)
at java.base@16/java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock.java:322)
at java.base@16/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:385)
at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388)
at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:702)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
30) Thread[id=9482, name=qtp153449517-9482, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
31) Thread[id=9570, name=SolrRrdBackendFactory-6588-thread-2, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
at java.base@16/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
at java.base@16/java.lang.Thread.run(Thread.java:831)
32) Thread[id=9541, name=SolrRrdBackendFactory-6619-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
at java.base@16/java.lang.Thread.run(Thread.java:831)
33) Thread[id=9533, name=Thread-1811, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/java.lang.Object.wait(Native Method)
at java.base@16/java.lang.Object.wait(Object.java:320)
at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2281)
34) Thread[id=9469, name=qtp1931265301-9469, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
35) Thread[id=9486, name=qtp153449517-9486, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
36) Thread[id=9453, name=qtp1312166695-9453, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
37) Thread[id=9491, name=qtp153449517-9491-acceptor-1@7dd30796-ServerConnector@5bbc5236{ssl, (ssl, alpn, http/1.1, h2)}{127.0.0.1:46045}, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:714)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:937)
at java.base@16/java.util.concurrent.locks.ReentrantLock$Sync.lock(ReentrantLock.java:153)
at java.base@16/java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock.java:322)
at java.base@16/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:385)
at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388)
at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:702)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
38) Thread[id=9470, name=qtp1931265301-9470, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
39) Thread[id=9467, name=qtp1931265301-9467, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
40) Thread[id=9515, name=MetricsHistoryHandler-6589-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/java.lang.Thread.sleep(Native Method)
at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:138)
at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:89)
at app//org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:351)
at app//org.apache.solr.client.solrj.impl.ZkDistribStateManager.getData(ZkDistribStateManager.java:88)
at app//org.apache.solr.client.solrj.cloud.DistribStateManager.getData(DistribStateManager.java:56)
at app//org.apache.solr.handler.admin.MetricsHistoryHandler.getOverseerLeader(MetricsHistoryHandler.java:335)
at app//org.apache.solr.handler.admin.MetricsHistoryHandler.amIOverseerLeader(MetricsHistoryHandler.java:373)
at app//org.apache.solr.handler.admin.MetricsHistoryHandler.amIOverseerLeader(MetricsHistoryHandler.java:368)
at app//org.apache.solr.handler.admin.MetricsHistoryHandler.collectGlobalMetrics(MetricsHistoryHandler.java:469)
at app//org.apache.solr.handler.admin.MetricsHistoryHandler.collectMetrics(MetricsHistoryHandler.java:395)
at app//org.apache.solr.handler.admin.MetricsHistoryHandler.lambda$new$0(MetricsHistoryHandler.java:242)
at app//org.apache.solr.handler.admin.MetricsHistoryHandler$$Lambda$954/0x0000000801361e60.run(Unknown Source)
at java.base@16/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
at java.base@16/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
at java.base@16/java.lang.Thread.run(Thread.java:831)
41) Thread[id=9516, name=Thread-1810, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/java.lang.Object.wait(Native Method)
at java.base@16/java.lang.Object.wait(Object.java:320)
at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2281)
42) Thread[id=9472, name=qtp1931265301-9472, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
43) Thread[id=9489, name=Session-HouseKeeper-63b1bc1d-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
at java.base@16/java.lang.Thread.run(Thread.java:831)
44) Thread[id=9456, name=qtp1312166695-9456, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
45) Thread[id=9455, name=qtp1312166695-9455, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
46) Thread[id=9564, name=qtp153449517-9564, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023)
at java.base@16/java.lang.Thread.run(Thread.java:831)
47) Thread[id=9458, name=qtp1312166695-9458, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
48) Thread[id=9490, name=qtp153449517-9490-acceptor-0@47b75565-ServerConnector@5bbc5236{ssl, (ssl, alpn, http/1.1, h2)}{127.0.0.1:46045}, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.Net.accept(Native Method)
at java.base@16/sun.nio.ch.ServerSocketChannelImpl.implAccept(ServerSocketChannelImpl.java:424)
at java.base@16/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:390)
at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388)
at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:702)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
49) Thread[id=9514, name=SolrRrdBackendFactory-6588-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
at java.base@16/java.lang.Thread.run(Thread.java:831)
50) Thread[id=9675, name=qtp1931265301-9675, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023)
at java.base@16/java.lang.Thread.run(Thread.java:831)
51) Thread[id=9543, name=Thread-1812, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/java.lang.Object.wait(Native Method)
at java.base@16/java.lang.Object.wait(Object.java:320)
at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2281)
52) Thread[id=9481, name=qtp153449517-9481, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
53) Thread[id=9488, name=qtp153449517-9488, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
54) Thread[id=9461, name=qtp1312166695-9461-acceptor-1@13d0161d-ServerConnector@401f7e19{ssl, (ssl, alpn, http/1.1, h2)}{127.0.0.1:40869}, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.Net.accept(Native Method)
at java.base@16/sun.nio.ch.ServerSocketChannelImpl.implAccept(ServerSocketChannelImpl.java:424)
at java.base@16/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:390)
at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388)
at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:702)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
55) Thread[id=9457, name=qtp1312166695-9457, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
at __randomizedtesting.SeedInfo.seed([940F3B336DBB6C96]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=9460, name=qtp1312166695-9460, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023) at java.base@16/java.lang.Thread.run(Thread.java:831) 2) Thread[id=9484, name=qtp153449517-9484, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 3) Thread[id=9571, name=SolrRrdBackendFactory-6609-thread-2, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505) at java.base@16/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at java.base@16/java.lang.Thread.run(Thread.java:831) 4) Thread[id=9451, name=qtp1312166695-9451, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 5) Thread[id=9563, name=qtp1312166695-9563, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023) at java.base@16/java.lang.Thread.run(Thread.java:831) 6) Thread[id=9565, name=Connector-Scheduler-5bbc5236-1, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505) at java.base@16/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at java.base@16/java.lang.Thread.run(Thread.java:831) 7) Thread[id=9454, name=qtp1312166695-9454, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 8) Thread[id=9532, name=MetricsHistoryHandler-6610-thread-1, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505) at java.base@16/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at java.base@16/java.lang.Thread.run(Thread.java:831) 9) Thread[id=9572, name=SolrRrdBackendFactory-6619-thread-2, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505) at java.base@16/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at java.base@16/java.lang.Thread.run(Thread.java:831) 10) Thread[id=9452, name=qtp1312166695-9452, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 11) Thread[id=9554, name=Connector-Scheduler-401f7e19-1, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505) at java.base@16/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at java.base@16/java.lang.Thread.run(Thread.java:831) 12) Thread[id=9531, name=SolrRrdBackendFactory-6609-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at java.base@16/java.lang.Thread.run(Thread.java:831) 13) Thread[id=9673, name=qtp153449517-9673, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023) at java.base@16/java.lang.Thread.run(Thread.java:831) 14) Thread[id=9483, name=qtp153449517-9483, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 15) Thread[id=9683, name=qtp1312166695-9683, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023) at java.base@16/java.lang.Thread.run(Thread.java:831) 16) Thread[id=9542, name=MetricsHistoryHandler-6620-thread-1, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505) at java.base@16/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at java.base@16/java.lang.Thread.run(Thread.java:831) 17) Thread[id=9459, name=Session-HouseKeeper-ff43509-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at java.base@16/java.lang.Thread.run(Thread.java:831) 18) Thread[id=9682, name=qtp1312166695-9682, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023) at java.base@16/java.lang.Thread.run(Thread.java:831) 19) Thread[id=9487, name=qtp153449517-9487, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 20) Thread[id=9485, name=qtp153449517-9485, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 21) Thread[id=9476, name=qtp1931265301-9476, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023) at java.base@16/java.lang.Thread.run(Thread.java:831) 22) Thread[id=9471, name=qtp1931265301-9471, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 23) Thread[id=9569, name=qtp1931265301-9569, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023) at java.base@16/java.lang.Thread.run(Thread.java:831) 24) Thread[id=9468, name=qtp1931265301-9468, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 25) Thread[id=9568, name=Connector-Scheduler-16f86070-1, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505) at java.base@16/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at java.base@16/java.lang.Thread.run(Thread.java:831) 26) Thread[id=9466, name=qtp1931265301-9466, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 27) Thread[id=9473, name=qtp1931265301-9473, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 28) Thread[id=9474, name=Session-HouseKeeper-2d64b24c-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at java.base@16/java.lang.Thread.run(Thread.java:831) 29) Thread[id=9475, name=qtp1931265301-9475, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023) at java.base@16/java.lang.Thread.run(Thread.java:831) 30) Thread[id=9482, name=qtp153449517-9482, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 31) Thread[id=9570, name=SolrRrdBackendFactory-6588-thread-2, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505) at java.base@16/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at java.base@16/java.lang.Thread.run(Thread.java:831) 32) Thread[id=9541, name=SolrRrdBackendFactory-6619-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at java.base@16/java.lang.Thread.run(Thread.java:831) 33) Thread[id=9533, name=Thread-1811, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/java.lang.Object.wait(Native Method) at java.base@16/java.lang.Object.wait(Object.java:320) at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2281) 34) Thread[id=9469, name=qtp1931265301-9469, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 35) Thread[id=9486, name=qtp153449517-9486, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 36) Thread[id=9453, name=qtp1312166695-9453, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 37) Thread[id=9491, name=qtp153449517-9491, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023) at java.base@16/java.lang.Thread.run(Thread.java:831) 38) Thread[id=9470, name=qtp1931265301-9470, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 39) Thread[id=9467, name=qtp1931265301-9467, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 40) Thread[id=9515, name=MetricsHistoryHandler-6589-thread-1, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505) at java.base@16/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at java.base@16/java.lang.Thread.run(Thread.java:831) 41) Thread[id=9516, name=Thread-1810, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/java.lang.Object.wait(Native Method) at java.base@16/java.lang.Object.wait(Object.java:320) at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2281) 42) Thread[id=9472, name=qtp1931265301-9472, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 43) Thread[id=9489, name=Session-HouseKeeper-63b1bc1d-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at java.base@16/java.lang.Thread.run(Thread.java:831) 44) Thread[id=9456, name=qtp1312166695-9456, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 45) Thread[id=9455, name=qtp1312166695-9455, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 46) Thread[id=9564, name=qtp153449517-9564, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023) at java.base@16/java.lang.Thread.run(Thread.java:831) 47) Thread[id=9458, name=qtp1312166695-9458, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 48) Thread[id=9490, name=qtp153449517-9490, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023) at java.base@16/java.lang.Thread.run(Thread.java:831) 49) Thread[id=9514, name=SolrRrdBackendFactory-6588-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at java.base@16/java.lang.Thread.run(Thread.java:831) 50) Thread[id=9675, name=qtp1931265301-9675, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023) at java.base@16/java.lang.Thread.run(Thread.java:831) 51) Thread[id=9543, name=Thread-1812, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/java.lang.Object.wait(Native Method) at java.base@16/java.lang.Object.wait(Object.java:320) at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2281) 52) Thread[id=9481, name=qtp153449517-9481, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 53) Thread[id=9488, name=qtp153449517-9488, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 54) Thread[id=9461, name=qtp1312166695-9461, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023) at java.base@16/java.lang.Thread.run(Thread.java:831) 55) Thread[id=9457, name=qtp1312166695-9457, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=9460, name=qtp1312166695-9460, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023)
at java.base@16/java.lang.Thread.run(Thread.java:831)
2) Thread[id=9484, name=qtp153449517-9484, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
3) Thread[id=9571, name=SolrRrdBackendFactory-6609-thread-2, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
at java.base@16/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
at java.base@16/java.lang.Thread.run(Thread.java:831)
4) Thread[id=9451, name=qtp1312166695-9451, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
5) Thread[id=9563, name=qtp1312166695-9563, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023)
at java.base@16/java.lang.Thread.run(Thread.java:831)
6) Thread[id=9565, name=Connector-Scheduler-5bbc5236-1, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
at java.base@16/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
at java.base@16/java.lang.Thread.run(Thread.java:831)
7) Thread[id=9454, name=qtp1312166695-9454, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
8) Thread[id=9532, name=MetricsHistoryHandler-6610-thread-1, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
at java.base@16/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
at java.base@16/java.lang.Thread.run(Thread.java:831)
9) Thread[id=9572, name=SolrRrdBackendFactory-6619-thread-2, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
at java.base@16/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
at java.base@16/java.lang.Thread.run(Thread.java:831)
10) Thread[id=9452, name=qtp1312166695-9452, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
11) Thread[id=9554, name=Connector-Scheduler-401f7e19-1, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
at java.base@16/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
at java.base@16/java.lang.Thread.run(Thread.java:831)
12) Thread[id=9531, name=SolrRrdBackendFactory-6609-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
at java.base@16/java.lang.Thread.run(Thread.java:831)
13) Thread[id=9673, name=qtp153449517-9673, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023)
at java.base@16/java.lang.Thread.run(Thread.java:831)
14) Thread[id=9483, name=qtp153449517-9483, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
15) Thread[id=9683, name=qtp1312166695-9683, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023)
at java.base@16/java.lang.Thread.run(Thread.java:831)
16) Thread[id=9542, name=MetricsHistoryHandler-6620-thread-1, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
at java.base@16/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
at java.base@16/java.lang.Thread.run(Thread.java:831)
17) Thread[id=9459, name=Session-HouseKeeper-ff43509-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
at java.base@16/java.lang.Thread.run(Thread.java:831)
18) Thread[id=9682, name=qtp1312166695-9682, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023)
at java.base@16/java.lang.Thread.run(Thread.java:831)
19) Thread[id=9487, name=qtp153449517-9487, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
20) Thread[id=9485, name=qtp153449517-9485, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
21) Thread[id=9476, name=qtp1931265301-9476, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023)
at java.base@16/java.lang.Thread.run(Thread.java:831)
22) Thread[id=9471, name=qtp1931265301-9471, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
23) Thread[id=9569, name=qtp1931265301-9569, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023)
at java.base@16/java.lang.Thread.run(Thread.java:831)
24) Thread[id=9468, name=qtp1931265301-9468, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
25) Thread[id=9568, name=Connector-Scheduler-16f86070-1, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
at java.base@16/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
at java.base@16/java.lang.Thread.run(Thread.java:831)
26) Thread[id=9466, name=qtp1931265301-9466, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
27) Thread[id=9473, name=qtp1931265301-9473, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
28) Thread[id=9474, name=Session-HouseKeeper-2d64b24c-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
at java.base@16/java.lang.Thread.run(Thread.java:831)
29) Thread[id=9475, name=qtp1931265301-9475, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023)
at java.base@16/java.lang.Thread.run(Thread.java:831)
30) Thread[id=9482, name=qtp153449517-9482, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
31) Thread[id=9570, name=SolrRrdBackendFactory-6588-thread-2, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
at java.base@16/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
at java.base@16/java.lang.Thread.run(Thread.java:831)
32) Thread[id=9541, name=SolrRrdBackendFactory-6619-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
at java.base@16/java.lang.Thread.run(Thread.java:831)
33) Thread[id=9533, name=Thread-1811, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/java.lang.Object.wait(Native Method)
at java.base@16/java.lang.Object.wait(Object.java:320)
at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2281)
34) Thread[id=9469, name=qtp1931265301-9469, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
35) Thread[id=9486, name=qtp153449517-9486, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
36) Thread[id=9453, name=qtp1312166695-9453, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
37) Thread[id=9491, name=qtp153449517-9491, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023)
at java.base@16/java.lang.Thread.run(Thread.java:831)
38) Thread[id=9470, name=qtp1931265301-9470, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
39) Thread[id=9467, name=qtp1931265301-9467, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
40) Thread[id=9515, name=MetricsHistoryHandler-6589-thread-1, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
at java.base@16/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
at java.base@16/java.lang.Thread.run(Thread.java:831)
41) Thread[id=9516, name=Thread-1810, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/java.lang.Object.wait(Native Method)
at java.base@16/java.lang.Object.wait(Object.java:320)
at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2281)
42) Thread[id=9472, name=qtp1931265301-9472, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
43) Thread[id=9489, name=Session-HouseKeeper-63b1bc1d-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
at java.base@16/java.lang.Thread.run(Thread.java:831)
44) Thread[id=9456, name=qtp1312166695-9456, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
45) Thread[id=9455, name=qtp1312166695-9455, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
46) Thread[id=9564, name=qtp153449517-9564, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023)
at java.base@16/java.lang.Thread.run(Thread.java:831)
47) Thread[id=9458, name=qtp1312166695-9458, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
48) Thread[id=9490, name=qtp153449517-9490, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023)
at java.base@16/java.lang.Thread.run(Thread.java:831)
49) Thread[id=9514, name=SolrRrdBackendFactory-6588-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
at java.base@16/java.lang.Thread.run(Thread.java:831)
50) Thread[id=9675, name=qtp1931265301-9675, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023)
at java.base@16/java.lang.Thread.run(Thread.java:831)
51) Thread[id=9543, name=Thread-1812, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/java.lang.Object.wait(Native Method)
at java.base@16/java.lang.Object.wait(Object.java:320)
at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2281)
52) Thread[id=9481, name=qtp153449517-9481, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
53) Thread[id=9488, name=qtp153449517-9488, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
54) Thread[id=9461, name=qtp1312166695-9461, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023)
at java.base@16/java.lang.Thread.run(Thread.java:831)
55) Thread[id=9457, name=qtp1312166695-9457, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
at __randomizedtesting.SeedInfo.seed([940F3B336DBB6C96]:0)
FAILED: org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest.test
Error Message:
Number of replicas in the state does not match what we set:1 vs 3
Stack Trace:
org.apache.solr.cloud.ZkController$NotInClusterStateException: Number of replicas in the state does not match what we set:1 vs 3
at __randomizedtesting.SeedInfo.seed([940F3B336DBB6C96:1C5B04E9C347016E]:0)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForActiveReplicaCount(AbstractFullDistribZkTestBase.java:635)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:574)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:360)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1088)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1061)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.base/java.lang.Thread.run(Thread.java:831)
FAILED: org.apache.solr.metrics.reporters.solr.SolrCloudReportersTest.testDefaultPlugins
Error Message:
Timeout occurred while waiting response from server at: https://127.0.0.1:43893/solr
Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occurred while waiting response from server at: https://127.0.0.1:43893/solr
at __randomizedtesting.SeedInfo.seed([940F3B336DBB6C96:DC8F69E331FB4FBB]:0)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:692)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:266)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
at org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:369)
at org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:297)
at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.sendRequest(BaseCloudSolrClient.java:1171)
at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.requestWithRetryOnStaleState(BaseCloudSolrClient.java:934)
at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.request(BaseCloudSolrClient.java:866)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:214)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:231)
at org.apache.solr.metrics.reporters.solr.SolrCloudReportersTest.testDefaultPlugins(SolrCloudReportersTest.java:176)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:78)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:567)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.base/java.lang.Thread.run(Thread.java:831)
Caused by: java.net.SocketTimeoutException: Read timed out
at java.base/sun.nio.ch.NioSocketImpl.timedRead(NioSocketImpl.java:283)
at java.base/sun.nio.ch.NioSocketImpl.implRead(NioSocketImpl.java:309)
at java.base/sun.nio.ch.NioSocketImpl.read(NioSocketImpl.java:350)
at java.base/sun.nio.ch.NioSocketImpl$1.read(NioSocketImpl.java:803)
at java.base/java.net.Socket$SocketInputStream.read(Socket.java:976)
at java.base/sun.security.ssl.SSLSocketInputRecord.read(SSLSocketInputRecord.java:478)
at java.base/sun.security.ssl.SSLSocketInputRecord.readHeader(SSLSocketInputRecord.java:472)
at java.base/sun.security.ssl.SSLSocketInputRecord.bytesInCompletePacket(SSLSocketInputRecord.java:70)
at java.base/sun.security.ssl.SSLSocketImpl.readApplicationRecord(SSLSocketImpl.java:1443)
at java.base/sun.security.ssl.SSLSocketImpl$AppInputStream.read(SSLSocketImpl.java:1047)
at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:280)
at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:157)
at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186)
at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:571)
... 52 more
[JENKINS] Lucene-Solr-8.9-Linux (64bit/jdk-16) - Build # 196 -
Still Unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-8.9-Linux/196/
Java: 64bit/jdk-16 -XX:+UseCompressedOops -XX:+UseG1GC
4 tests failed.
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest
Error Message:
55 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest: 1) Thread[id=9460, name=qtp1312166695-9460-acceptor-0@486ac7a4-ServerConnector@401f7e19{ssl, (ssl, alpn, http/1.1, h2)}{127.0.0.1:40869}, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:714) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:937) at java.base@16/java.util.concurrent.locks.ReentrantLock$Sync.lock(ReentrantLock.java:153) at java.base@16/java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock.java:322) at java.base@16/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:385) at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388) at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:702) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 2) Thread[id=9484, name=qtp153449517-9484, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 3) Thread[id=9571, name=SolrRrdBackendFactory-6609-thread-2, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505) at java.base@16/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at java.base@16/java.lang.Thread.run(Thread.java:831) 4) Thread[id=9451, name=qtp1312166695-9451, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 5) Thread[id=9563, name=qtp1312166695-9563, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023) at java.base@16/java.lang.Thread.run(Thread.java:831) 6) Thread[id=9565, name=Connector-Scheduler-5bbc5236-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at java.base@16/java.lang.Thread.run(Thread.java:831) 7) Thread[id=9454, name=qtp1312166695-9454, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 8) Thread[id=9532, name=MetricsHistoryHandler-6610-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/java.lang.Thread.sleep(Native Method) at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:138) at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:89) at app//org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:351) at app//org.apache.solr.client.solrj.impl.ZkDistribStateManager.getData(ZkDistribStateManager.java:88) at app//org.apache.solr.client.solrj.cloud.DistribStateManager.getData(DistribStateManager.java:56) at app//org.apache.solr.handler.admin.MetricsHistoryHandler.getOverseerLeader(MetricsHistoryHandler.java:335) at app//org.apache.solr.handler.admin.MetricsHistoryHandler.amIOverseerLeader(MetricsHistoryHandler.java:373) at app//org.apache.solr.handler.admin.MetricsHistoryHandler.amIOverseerLeader(MetricsHistoryHandler.java:368) at app//org.apache.solr.handler.admin.MetricsHistoryHandler.collectGlobalMetrics(MetricsHistoryHandler.java:469) at app//org.apache.solr.handler.admin.MetricsHistoryHandler.collectMetrics(MetricsHistoryHandler.java:395) at app//org.apache.solr.handler.admin.MetricsHistoryHandler.lambda$new$0(MetricsHistoryHandler.java:242) at app//org.apache.solr.handler.admin.MetricsHistoryHandler$$Lambda$954/0x0000000801361e60.run(Unknown Source) at java.base@16/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base@16/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at java.base@16/java.lang.Thread.run(Thread.java:831) 9) Thread[id=9572, name=SolrRrdBackendFactory-6619-thread-2, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505) at java.base@16/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at java.base@16/java.lang.Thread.run(Thread.java:831) 10) Thread[id=9452, name=qtp1312166695-9452, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 11) Thread[id=9554, name=Connector-Scheduler-401f7e19-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at java.base@16/java.lang.Thread.run(Thread.java:831) 12) Thread[id=9531, name=SolrRrdBackendFactory-6609-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at java.base@16/java.lang.Thread.run(Thread.java:831) 13) Thread[id=9673, name=qtp153449517-9673, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023) at java.base@16/java.lang.Thread.run(Thread.java:831) 14) Thread[id=9483, name=qtp153449517-9483, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 15) Thread[id=9683, name=qtp1312166695-9683, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023) at java.base@16/java.lang.Thread.run(Thread.java:831) 16) Thread[id=9542, name=MetricsHistoryHandler-6620-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/java.lang.Thread.sleep(Native Method) at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:138) at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:89) at app//org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:351) at app//org.apache.solr.client.solrj.impl.ZkDistribStateManager.getData(ZkDistribStateManager.java:88) at app//org.apache.solr.client.solrj.cloud.DistribStateManager.getData(DistribStateManager.java:56) at app//org.apache.solr.handler.admin.MetricsHistoryHandler.getOverseerLeader(MetricsHistoryHandler.java:335) at app//org.apache.solr.handler.admin.MetricsHistoryHandler.amIOverseerLeader(MetricsHistoryHandler.java:373) at app//org.apache.solr.handler.admin.MetricsHistoryHandler.amIOverseerLeader(MetricsHistoryHandler.java:368) at app//org.apache.solr.handler.admin.MetricsHistoryHandler.collectGlobalMetrics(MetricsHistoryHandler.java:469) at app//org.apache.solr.handler.admin.MetricsHistoryHandler.collectMetrics(MetricsHistoryHandler.java:395) at app//org.apache.solr.handler.admin.MetricsHistoryHandler.lambda$new$0(MetricsHistoryHandler.java:242) at app//org.apache.solr.handler.admin.MetricsHistoryHandler$$Lambda$954/0x0000000801361e60.run(Unknown Source) at java.base@16/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base@16/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at java.base@16/java.lang.Thread.run(Thread.java:831) 17) Thread[id=9459, name=Session-HouseKeeper-ff43509-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at java.base@16/java.lang.Thread.run(Thread.java:831) 18) Thread[id=9682, name=qtp1312166695-9682, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023) at java.base@16/java.lang.Thread.run(Thread.java:831) 19) Thread[id=9487, name=qtp153449517-9487, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 20) Thread[id=9485, name=qtp153449517-9485, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 21) Thread[id=9476, name=qtp1931265301-9476-acceptor-1@11a1a8e1-ServerConnector@16f86070{ssl, (ssl, alpn, http/1.1, h2)}{127.0.0.1:45021}, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.Net.accept(Native Method) at java.base@16/sun.nio.ch.ServerSocketChannelImpl.implAccept(ServerSocketChannelImpl.java:424) at java.base@16/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:390) at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388) at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:702) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 22) Thread[id=9471, name=qtp1931265301-9471, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 23) Thread[id=9569, name=qtp1931265301-9569, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023) at java.base@16/java.lang.Thread.run(Thread.java:831) 24) Thread[id=9468, name=qtp1931265301-9468, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 25) Thread[id=9568, name=Connector-Scheduler-16f86070-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at java.base@16/java.lang.Thread.run(Thread.java:831) 26) Thread[id=9466, name=qtp1931265301-9466, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 27) Thread[id=9473, name=qtp1931265301-9473, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 28) Thread[id=9474, name=Session-HouseKeeper-2d64b24c-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at java.base@16/java.lang.Thread.run(Thread.java:831) 29) Thread[id=9475, name=qtp1931265301-9475-acceptor-0@64e0dff8-ServerConnector@16f86070{ssl, (ssl, alpn, http/1.1, h2)}{127.0.0.1:45021}, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:714) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:937) at java.base@16/java.util.concurrent.locks.ReentrantLock$Sync.lock(ReentrantLock.java:153) at java.base@16/java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock.java:322) at java.base@16/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:385) at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388) at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:702) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 30) Thread[id=9482, name=qtp153449517-9482, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 31) Thread[id=9570, name=SolrRrdBackendFactory-6588-thread-2, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505) at java.base@16/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at java.base@16/java.lang.Thread.run(Thread.java:831) 32) Thread[id=9541, name=SolrRrdBackendFactory-6619-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at java.base@16/java.lang.Thread.run(Thread.java:831) 33) Thread[id=9533, name=Thread-1811, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/java.lang.Object.wait(Native Method) at java.base@16/java.lang.Object.wait(Object.java:320) at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2281) 34) Thread[id=9469, name=qtp1931265301-9469, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 35) Thread[id=9486, name=qtp153449517-9486, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 36) Thread[id=9453, name=qtp1312166695-9453, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 37) Thread[id=9491, name=qtp153449517-9491-acceptor-1@7dd30796-ServerConnector@5bbc5236{ssl, (ssl, alpn, http/1.1, h2)}{127.0.0.1:46045}, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:714) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:937) at java.base@16/java.util.concurrent.locks.ReentrantLock$Sync.lock(ReentrantLock.java:153) at java.base@16/java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock.java:322) at java.base@16/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:385) at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388) at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:702) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 38) Thread[id=9470, name=qtp1931265301-9470, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 39) Thread[id=9467, name=qtp1931265301-9467, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 40) Thread[id=9515, name=MetricsHistoryHandler-6589-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/java.lang.Thread.sleep(Native Method) at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:138) at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:89) at app//org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:351) at app//org.apache.solr.client.solrj.impl.ZkDistribStateManager.getData(ZkDistribStateManager.java:88) at app//org.apache.solr.client.solrj.cloud.DistribStateManager.getData(DistribStateManager.java:56) at app//org.apache.solr.handler.admin.MetricsHistoryHandler.getOverseerLeader(MetricsHistoryHandler.java:335) at app//org.apache.solr.handler.admin.MetricsHistoryHandler.amIOverseerLeader(MetricsHistoryHandler.java:373) at app//org.apache.solr.handler.admin.MetricsHistoryHandler.amIOverseerLeader(MetricsHistoryHandler.java:368) at app//org.apache.solr.handler.admin.MetricsHistoryHandler.collectGlobalMetrics(MetricsHistoryHandler.java:469) at app//org.apache.solr.handler.admin.MetricsHistoryHandler.collectMetrics(MetricsHistoryHandler.java:395) at app//org.apache.solr.handler.admin.MetricsHistoryHandler.lambda$new$0(MetricsHistoryHandler.java:242) at app//org.apache.solr.handler.admin.MetricsHistoryHandler$$Lambda$954/0x0000000801361e60.run(Unknown Source) at java.base@16/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) at java.base@16/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130) at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at java.base@16/java.lang.Thread.run(Thread.java:831) 41) Thread[id=9516, name=Thread-1810, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/java.lang.Object.wait(Native Method) at java.base@16/java.lang.Object.wait(Object.java:320) at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2281) 42) Thread[id=9472, name=qtp1931265301-9472, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 43) Thread[id=9489, name=Session-HouseKeeper-63b1bc1d-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at java.base@16/java.lang.Thread.run(Thread.java:831) 44) Thread[id=9456, name=qtp1312166695-9456, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 45) Thread[id=9455, name=qtp1312166695-9455, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 46) Thread[id=9564, name=qtp153449517-9564, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023) at java.base@16/java.lang.Thread.run(Thread.java:831) 47) Thread[id=9458, name=qtp1312166695-9458, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 48) Thread[id=9490, name=qtp153449517-9490-acceptor-0@47b75565-ServerConnector@5bbc5236{ssl, (ssl, alpn, http/1.1, h2)}{127.0.0.1:46045}, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.Net.accept(Native Method) at java.base@16/sun.nio.ch.ServerSocketChannelImpl.implAccept(ServerSocketChannelImpl.java:424) at java.base@16/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:390) at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388) at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:702) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 49) Thread[id=9514, name=SolrRrdBackendFactory-6588-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at java.base@16/java.lang.Thread.run(Thread.java:831) 50) Thread[id=9675, name=qtp1931265301-9675, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023) at java.base@16/java.lang.Thread.run(Thread.java:831) 51) Thread[id=9543, name=Thread-1812, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/java.lang.Object.wait(Native Method) at java.base@16/java.lang.Object.wait(Object.java:320) at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2281) 52) Thread[id=9481, name=qtp153449517-9481, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 53) Thread[id=9488, name=qtp153449517-9488, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 54) Thread[id=9461, name=qtp1312166695-9461-acceptor-1@13d0161d-ServerConnector@401f7e19{ssl, (ssl, alpn, http/1.1, h2)}{127.0.0.1:40869}, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.Net.accept(Native Method) at java.base@16/sun.nio.ch.ServerSocketChannelImpl.implAccept(ServerSocketChannelImpl.java:424) at java.base@16/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:390) at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388) at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:702) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 55) Thread[id=9457, name=qtp1312166695-9457, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 55 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest:
1) Thread[id=9460, name=qtp1312166695-9460-acceptor-0@486ac7a4-ServerConnector@401f7e19{ssl, (ssl, alpn, http/1.1, h2)}{127.0.0.1:40869}, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:714)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:937)
at java.base@16/java.util.concurrent.locks.ReentrantLock$Sync.lock(ReentrantLock.java:153)
at java.base@16/java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock.java:322)
at java.base@16/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:385)
at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388)
at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:702)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
2) Thread[id=9484, name=qtp153449517-9484, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
3) Thread[id=9571, name=SolrRrdBackendFactory-6609-thread-2, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
at java.base@16/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
at java.base@16/java.lang.Thread.run(Thread.java:831)
4) Thread[id=9451, name=qtp1312166695-9451, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
5) Thread[id=9563, name=qtp1312166695-9563, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023)
at java.base@16/java.lang.Thread.run(Thread.java:831)
6) Thread[id=9565, name=Connector-Scheduler-5bbc5236-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
at java.base@16/java.lang.Thread.run(Thread.java:831)
7) Thread[id=9454, name=qtp1312166695-9454, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
8) Thread[id=9532, name=MetricsHistoryHandler-6610-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/java.lang.Thread.sleep(Native Method)
at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:138)
at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:89)
at app//org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:351)
at app//org.apache.solr.client.solrj.impl.ZkDistribStateManager.getData(ZkDistribStateManager.java:88)
at app//org.apache.solr.client.solrj.cloud.DistribStateManager.getData(DistribStateManager.java:56)
at app//org.apache.solr.handler.admin.MetricsHistoryHandler.getOverseerLeader(MetricsHistoryHandler.java:335)
at app//org.apache.solr.handler.admin.MetricsHistoryHandler.amIOverseerLeader(MetricsHistoryHandler.java:373)
at app//org.apache.solr.handler.admin.MetricsHistoryHandler.amIOverseerLeader(MetricsHistoryHandler.java:368)
at app//org.apache.solr.handler.admin.MetricsHistoryHandler.collectGlobalMetrics(MetricsHistoryHandler.java:469)
at app//org.apache.solr.handler.admin.MetricsHistoryHandler.collectMetrics(MetricsHistoryHandler.java:395)
at app//org.apache.solr.handler.admin.MetricsHistoryHandler.lambda$new$0(MetricsHistoryHandler.java:242)
at app//org.apache.solr.handler.admin.MetricsHistoryHandler$$Lambda$954/0x0000000801361e60.run(Unknown Source)
at java.base@16/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
at java.base@16/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
at java.base@16/java.lang.Thread.run(Thread.java:831)
9) Thread[id=9572, name=SolrRrdBackendFactory-6619-thread-2, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
at java.base@16/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
at java.base@16/java.lang.Thread.run(Thread.java:831)
10) Thread[id=9452, name=qtp1312166695-9452, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
11) Thread[id=9554, name=Connector-Scheduler-401f7e19-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
at java.base@16/java.lang.Thread.run(Thread.java:831)
12) Thread[id=9531, name=SolrRrdBackendFactory-6609-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
at java.base@16/java.lang.Thread.run(Thread.java:831)
13) Thread[id=9673, name=qtp153449517-9673, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023)
at java.base@16/java.lang.Thread.run(Thread.java:831)
14) Thread[id=9483, name=qtp153449517-9483, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
15) Thread[id=9683, name=qtp1312166695-9683, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023)
at java.base@16/java.lang.Thread.run(Thread.java:831)
16) Thread[id=9542, name=MetricsHistoryHandler-6620-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/java.lang.Thread.sleep(Native Method)
at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:138)
at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:89)
at app//org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:351)
at app//org.apache.solr.client.solrj.impl.ZkDistribStateManager.getData(ZkDistribStateManager.java:88)
at app//org.apache.solr.client.solrj.cloud.DistribStateManager.getData(DistribStateManager.java:56)
at app//org.apache.solr.handler.admin.MetricsHistoryHandler.getOverseerLeader(MetricsHistoryHandler.java:335)
at app//org.apache.solr.handler.admin.MetricsHistoryHandler.amIOverseerLeader(MetricsHistoryHandler.java:373)
at app//org.apache.solr.handler.admin.MetricsHistoryHandler.amIOverseerLeader(MetricsHistoryHandler.java:368)
at app//org.apache.solr.handler.admin.MetricsHistoryHandler.collectGlobalMetrics(MetricsHistoryHandler.java:469)
at app//org.apache.solr.handler.admin.MetricsHistoryHandler.collectMetrics(MetricsHistoryHandler.java:395)
at app//org.apache.solr.handler.admin.MetricsHistoryHandler.lambda$new$0(MetricsHistoryHandler.java:242)
at app//org.apache.solr.handler.admin.MetricsHistoryHandler$$Lambda$954/0x0000000801361e60.run(Unknown Source)
at java.base@16/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
at java.base@16/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
at java.base@16/java.lang.Thread.run(Thread.java:831)
17) Thread[id=9459, name=Session-HouseKeeper-ff43509-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
at java.base@16/java.lang.Thread.run(Thread.java:831)
18) Thread[id=9682, name=qtp1312166695-9682, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023)
at java.base@16/java.lang.Thread.run(Thread.java:831)
19) Thread[id=9487, name=qtp153449517-9487, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
20) Thread[id=9485, name=qtp153449517-9485, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
21) Thread[id=9476, name=qtp1931265301-9476-acceptor-1@11a1a8e1-ServerConnector@16f86070{ssl, (ssl, alpn, http/1.1, h2)}{127.0.0.1:45021}, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.Net.accept(Native Method)
at java.base@16/sun.nio.ch.ServerSocketChannelImpl.implAccept(ServerSocketChannelImpl.java:424)
at java.base@16/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:390)
at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388)
at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:702)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
22) Thread[id=9471, name=qtp1931265301-9471, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
23) Thread[id=9569, name=qtp1931265301-9569, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023)
at java.base@16/java.lang.Thread.run(Thread.java:831)
24) Thread[id=9468, name=qtp1931265301-9468, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
25) Thread[id=9568, name=Connector-Scheduler-16f86070-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
at java.base@16/java.lang.Thread.run(Thread.java:831)
26) Thread[id=9466, name=qtp1931265301-9466, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
27) Thread[id=9473, name=qtp1931265301-9473, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
28) Thread[id=9474, name=Session-HouseKeeper-2d64b24c-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
at java.base@16/java.lang.Thread.run(Thread.java:831)
29) Thread[id=9475, name=qtp1931265301-9475-acceptor-0@64e0dff8-ServerConnector@16f86070{ssl, (ssl, alpn, http/1.1, h2)}{127.0.0.1:45021}, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:714)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:937)
at java.base@16/java.util.concurrent.locks.ReentrantLock$Sync.lock(ReentrantLock.java:153)
at java.base@16/java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock.java:322)
at java.base@16/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:385)
at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388)
at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:702)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
30) Thread[id=9482, name=qtp153449517-9482, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
31) Thread[id=9570, name=SolrRrdBackendFactory-6588-thread-2, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
at java.base@16/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
at java.base@16/java.lang.Thread.run(Thread.java:831)
32) Thread[id=9541, name=SolrRrdBackendFactory-6619-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
at java.base@16/java.lang.Thread.run(Thread.java:831)
33) Thread[id=9533, name=Thread-1811, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/java.lang.Object.wait(Native Method)
at java.base@16/java.lang.Object.wait(Object.java:320)
at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2281)
34) Thread[id=9469, name=qtp1931265301-9469, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
35) Thread[id=9486, name=qtp153449517-9486, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
36) Thread[id=9453, name=qtp1312166695-9453, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
37) Thread[id=9491, name=qtp153449517-9491-acceptor-1@7dd30796-ServerConnector@5bbc5236{ssl, (ssl, alpn, http/1.1, h2)}{127.0.0.1:46045}, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:714)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:937)
at java.base@16/java.util.concurrent.locks.ReentrantLock$Sync.lock(ReentrantLock.java:153)
at java.base@16/java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock.java:322)
at java.base@16/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:385)
at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388)
at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:702)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
38) Thread[id=9470, name=qtp1931265301-9470, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
39) Thread[id=9467, name=qtp1931265301-9467, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
40) Thread[id=9515, name=MetricsHistoryHandler-6589-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/java.lang.Thread.sleep(Native Method)
at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:138)
at app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:89)
at app//org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:351)
at app//org.apache.solr.client.solrj.impl.ZkDistribStateManager.getData(ZkDistribStateManager.java:88)
at app//org.apache.solr.client.solrj.cloud.DistribStateManager.getData(DistribStateManager.java:56)
at app//org.apache.solr.handler.admin.MetricsHistoryHandler.getOverseerLeader(MetricsHistoryHandler.java:335)
at app//org.apache.solr.handler.admin.MetricsHistoryHandler.amIOverseerLeader(MetricsHistoryHandler.java:373)
at app//org.apache.solr.handler.admin.MetricsHistoryHandler.amIOverseerLeader(MetricsHistoryHandler.java:368)
at app//org.apache.solr.handler.admin.MetricsHistoryHandler.collectGlobalMetrics(MetricsHistoryHandler.java:469)
at app//org.apache.solr.handler.admin.MetricsHistoryHandler.collectMetrics(MetricsHistoryHandler.java:395)
at app//org.apache.solr.handler.admin.MetricsHistoryHandler.lambda$new$0(MetricsHistoryHandler.java:242)
at app//org.apache.solr.handler.admin.MetricsHistoryHandler$$Lambda$954/0x0000000801361e60.run(Unknown Source)
at java.base@16/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
at java.base@16/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
at java.base@16/java.lang.Thread.run(Thread.java:831)
41) Thread[id=9516, name=Thread-1810, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/java.lang.Object.wait(Native Method)
at java.base@16/java.lang.Object.wait(Object.java:320)
at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2281)
42) Thread[id=9472, name=qtp1931265301-9472, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
43) Thread[id=9489, name=Session-HouseKeeper-63b1bc1d-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
at java.base@16/java.lang.Thread.run(Thread.java:831)
44) Thread[id=9456, name=qtp1312166695-9456, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
45) Thread[id=9455, name=qtp1312166695-9455, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
46) Thread[id=9564, name=qtp153449517-9564, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023)
at java.base@16/java.lang.Thread.run(Thread.java:831)
47) Thread[id=9458, name=qtp1312166695-9458, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
48) Thread[id=9490, name=qtp153449517-9490-acceptor-0@47b75565-ServerConnector@5bbc5236{ssl, (ssl, alpn, http/1.1, h2)}{127.0.0.1:46045}, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.Net.accept(Native Method)
at java.base@16/sun.nio.ch.ServerSocketChannelImpl.implAccept(ServerSocketChannelImpl.java:424)
at java.base@16/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:390)
at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388)
at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:702)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
49) Thread[id=9514, name=SolrRrdBackendFactory-6588-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
at java.base@16/java.lang.Thread.run(Thread.java:831)
50) Thread[id=9675, name=qtp1931265301-9675, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023)
at java.base@16/java.lang.Thread.run(Thread.java:831)
51) Thread[id=9543, name=Thread-1812, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/java.lang.Object.wait(Native Method)
at java.base@16/java.lang.Object.wait(Object.java:320)
at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2281)
52) Thread[id=9481, name=qtp153449517-9481, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
53) Thread[id=9488, name=qtp153449517-9488, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
54) Thread[id=9461, name=qtp1312166695-9461-acceptor-1@13d0161d-ServerConnector@401f7e19{ssl, (ssl, alpn, http/1.1, h2)}{127.0.0.1:40869}, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.Net.accept(Native Method)
at java.base@16/sun.nio.ch.ServerSocketChannelImpl.implAccept(ServerSocketChannelImpl.java:424)
at java.base@16/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:390)
at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388)
at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:702)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
55) Thread[id=9457, name=qtp1312166695-9457, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
at __randomizedtesting.SeedInfo.seed([940F3B336DBB6C96]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=9460, name=qtp1312166695-9460, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023) at java.base@16/java.lang.Thread.run(Thread.java:831) 2) Thread[id=9484, name=qtp153449517-9484, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 3) Thread[id=9571, name=SolrRrdBackendFactory-6609-thread-2, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505) at java.base@16/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at java.base@16/java.lang.Thread.run(Thread.java:831) 4) Thread[id=9451, name=qtp1312166695-9451, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 5) Thread[id=9563, name=qtp1312166695-9563, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023) at java.base@16/java.lang.Thread.run(Thread.java:831) 6) Thread[id=9565, name=Connector-Scheduler-5bbc5236-1, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505) at java.base@16/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at java.base@16/java.lang.Thread.run(Thread.java:831) 7) Thread[id=9454, name=qtp1312166695-9454, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 8) Thread[id=9532, name=MetricsHistoryHandler-6610-thread-1, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505) at java.base@16/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at java.base@16/java.lang.Thread.run(Thread.java:831) 9) Thread[id=9572, name=SolrRrdBackendFactory-6619-thread-2, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505) at java.base@16/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at java.base@16/java.lang.Thread.run(Thread.java:831) 10) Thread[id=9452, name=qtp1312166695-9452, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 11) Thread[id=9554, name=Connector-Scheduler-401f7e19-1, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505) at java.base@16/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at java.base@16/java.lang.Thread.run(Thread.java:831) 12) Thread[id=9531, name=SolrRrdBackendFactory-6609-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at java.base@16/java.lang.Thread.run(Thread.java:831) 13) Thread[id=9673, name=qtp153449517-9673, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023) at java.base@16/java.lang.Thread.run(Thread.java:831) 14) Thread[id=9483, name=qtp153449517-9483, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 15) Thread[id=9683, name=qtp1312166695-9683, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023) at java.base@16/java.lang.Thread.run(Thread.java:831) 16) Thread[id=9542, name=MetricsHistoryHandler-6620-thread-1, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505) at java.base@16/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at java.base@16/java.lang.Thread.run(Thread.java:831) 17) Thread[id=9459, name=Session-HouseKeeper-ff43509-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at java.base@16/java.lang.Thread.run(Thread.java:831) 18) Thread[id=9682, name=qtp1312166695-9682, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023) at java.base@16/java.lang.Thread.run(Thread.java:831) 19) Thread[id=9487, name=qtp153449517-9487, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 20) Thread[id=9485, name=qtp153449517-9485, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 21) Thread[id=9476, name=qtp1931265301-9476, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023) at java.base@16/java.lang.Thread.run(Thread.java:831) 22) Thread[id=9471, name=qtp1931265301-9471, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 23) Thread[id=9569, name=qtp1931265301-9569, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023) at java.base@16/java.lang.Thread.run(Thread.java:831) 24) Thread[id=9468, name=qtp1931265301-9468, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 25) Thread[id=9568, name=Connector-Scheduler-16f86070-1, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505) at java.base@16/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at java.base@16/java.lang.Thread.run(Thread.java:831) 26) Thread[id=9466, name=qtp1931265301-9466, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 27) Thread[id=9473, name=qtp1931265301-9473, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 28) Thread[id=9474, name=Session-HouseKeeper-2d64b24c-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at java.base@16/java.lang.Thread.run(Thread.java:831) 29) Thread[id=9475, name=qtp1931265301-9475, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023) at java.base@16/java.lang.Thread.run(Thread.java:831) 30) Thread[id=9482, name=qtp153449517-9482, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 31) Thread[id=9570, name=SolrRrdBackendFactory-6588-thread-2, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505) at java.base@16/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at java.base@16/java.lang.Thread.run(Thread.java:831) 32) Thread[id=9541, name=SolrRrdBackendFactory-6619-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at java.base@16/java.lang.Thread.run(Thread.java:831) 33) Thread[id=9533, name=Thread-1811, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/java.lang.Object.wait(Native Method) at java.base@16/java.lang.Object.wait(Object.java:320) at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2281) 34) Thread[id=9469, name=qtp1931265301-9469, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 35) Thread[id=9486, name=qtp153449517-9486, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 36) Thread[id=9453, name=qtp1312166695-9453, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 37) Thread[id=9491, name=qtp153449517-9491, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023) at java.base@16/java.lang.Thread.run(Thread.java:831) 38) Thread[id=9470, name=qtp1931265301-9470, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 39) Thread[id=9467, name=qtp1931265301-9467, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 40) Thread[id=9515, name=MetricsHistoryHandler-6589-thread-1, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505) at java.base@16/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at java.base@16/java.lang.Thread.run(Thread.java:831) 41) Thread[id=9516, name=Thread-1810, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/java.lang.Object.wait(Native Method) at java.base@16/java.lang.Object.wait(Object.java:320) at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2281) 42) Thread[id=9472, name=qtp1931265301-9472, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 43) Thread[id=9489, name=Session-HouseKeeper-63b1bc1d-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at java.base@16/java.lang.Thread.run(Thread.java:831) 44) Thread[id=9456, name=qtp1312166695-9456, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 45) Thread[id=9455, name=qtp1312166695-9455, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 46) Thread[id=9564, name=qtp153449517-9564, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023) at java.base@16/java.lang.Thread.run(Thread.java:831) 47) Thread[id=9458, name=qtp1312166695-9458, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 48) Thread[id=9490, name=qtp153449517-9490, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023) at java.base@16/java.lang.Thread.run(Thread.java:831) 49) Thread[id=9514, name=SolrRrdBackendFactory-6588-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056) at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116) at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630) at java.base@16/java.lang.Thread.run(Thread.java:831) 50) Thread[id=9675, name=qtp1931265301-9675, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023) at java.base@16/java.lang.Thread.run(Thread.java:831) 51) Thread[id=9543, name=Thread-1812, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/java.lang.Object.wait(Native Method) at java.base@16/java.lang.Object.wait(Object.java:320) at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2281) 52) Thread[id=9481, name=qtp153449517-9481, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 53) Thread[id=9488, name=qtp153449517-9488, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831) 54) Thread[id=9461, name=qtp1312166695-9461, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/jdk.internal.misc.Unsafe.park(Native Method) at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023) at java.base@16/java.lang.Thread.run(Thread.java:831) 55) Thread[id=9457, name=qtp1312166695-9457, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest] at java.base@16/sun.nio.ch.EPoll.wait(Native Method) at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120) at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036) at java.base@16/java.lang.Thread.run(Thread.java:831)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=9460, name=qtp1312166695-9460, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023)
at java.base@16/java.lang.Thread.run(Thread.java:831)
2) Thread[id=9484, name=qtp153449517-9484, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
3) Thread[id=9571, name=SolrRrdBackendFactory-6609-thread-2, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
at java.base@16/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
at java.base@16/java.lang.Thread.run(Thread.java:831)
4) Thread[id=9451, name=qtp1312166695-9451, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
5) Thread[id=9563, name=qtp1312166695-9563, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023)
at java.base@16/java.lang.Thread.run(Thread.java:831)
6) Thread[id=9565, name=Connector-Scheduler-5bbc5236-1, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
at java.base@16/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
at java.base@16/java.lang.Thread.run(Thread.java:831)
7) Thread[id=9454, name=qtp1312166695-9454, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
8) Thread[id=9532, name=MetricsHistoryHandler-6610-thread-1, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
at java.base@16/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
at java.base@16/java.lang.Thread.run(Thread.java:831)
9) Thread[id=9572, name=SolrRrdBackendFactory-6619-thread-2, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
at java.base@16/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
at java.base@16/java.lang.Thread.run(Thread.java:831)
10) Thread[id=9452, name=qtp1312166695-9452, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
11) Thread[id=9554, name=Connector-Scheduler-401f7e19-1, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
at java.base@16/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
at java.base@16/java.lang.Thread.run(Thread.java:831)
12) Thread[id=9531, name=SolrRrdBackendFactory-6609-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
at java.base@16/java.lang.Thread.run(Thread.java:831)
13) Thread[id=9673, name=qtp153449517-9673, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023)
at java.base@16/java.lang.Thread.run(Thread.java:831)
14) Thread[id=9483, name=qtp153449517-9483, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
15) Thread[id=9683, name=qtp1312166695-9683, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023)
at java.base@16/java.lang.Thread.run(Thread.java:831)
16) Thread[id=9542, name=MetricsHistoryHandler-6620-thread-1, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
at java.base@16/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
at java.base@16/java.lang.Thread.run(Thread.java:831)
17) Thread[id=9459, name=Session-HouseKeeper-ff43509-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
at java.base@16/java.lang.Thread.run(Thread.java:831)
18) Thread[id=9682, name=qtp1312166695-9682, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023)
at java.base@16/java.lang.Thread.run(Thread.java:831)
19) Thread[id=9487, name=qtp153449517-9487, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
20) Thread[id=9485, name=qtp153449517-9485, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
21) Thread[id=9476, name=qtp1931265301-9476, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023)
at java.base@16/java.lang.Thread.run(Thread.java:831)
22) Thread[id=9471, name=qtp1931265301-9471, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
23) Thread[id=9569, name=qtp1931265301-9569, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023)
at java.base@16/java.lang.Thread.run(Thread.java:831)
24) Thread[id=9468, name=qtp1931265301-9468, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
25) Thread[id=9568, name=Connector-Scheduler-16f86070-1, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
at java.base@16/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
at java.base@16/java.lang.Thread.run(Thread.java:831)
26) Thread[id=9466, name=qtp1931265301-9466, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
27) Thread[id=9473, name=qtp1931265301-9473, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
28) Thread[id=9474, name=Session-HouseKeeper-2d64b24c-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
at java.base@16/java.lang.Thread.run(Thread.java:831)
29) Thread[id=9475, name=qtp1931265301-9475, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023)
at java.base@16/java.lang.Thread.run(Thread.java:831)
30) Thread[id=9482, name=qtp153449517-9482, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
31) Thread[id=9570, name=SolrRrdBackendFactory-6588-thread-2, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
at java.base@16/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
at java.base@16/java.lang.Thread.run(Thread.java:831)
32) Thread[id=9541, name=SolrRrdBackendFactory-6619-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
at java.base@16/java.lang.Thread.run(Thread.java:831)
33) Thread[id=9533, name=Thread-1811, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/java.lang.Object.wait(Native Method)
at java.base@16/java.lang.Object.wait(Object.java:320)
at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2281)
34) Thread[id=9469, name=qtp1931265301-9469, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
35) Thread[id=9486, name=qtp153449517-9486, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
36) Thread[id=9453, name=qtp1312166695-9453, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
37) Thread[id=9491, name=qtp153449517-9491, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023)
at java.base@16/java.lang.Thread.run(Thread.java:831)
38) Thread[id=9470, name=qtp1931265301-9470, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
39) Thread[id=9467, name=qtp1931265301-9467, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
40) Thread[id=9515, name=MetricsHistoryHandler-6589-thread-1, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
at java.base@16/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
at java.base@16/java.lang.Thread.run(Thread.java:831)
41) Thread[id=9516, name=Thread-1810, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/java.lang.Object.wait(Native Method)
at java.base@16/java.lang.Object.wait(Object.java:320)
at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2281)
42) Thread[id=9472, name=qtp1931265301-9472, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
43) Thread[id=9489, name=Session-HouseKeeper-63b1bc1d-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
at java.base@16/java.lang.Thread.run(Thread.java:831)
44) Thread[id=9456, name=qtp1312166695-9456, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
45) Thread[id=9455, name=qtp1312166695-9455, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
46) Thread[id=9564, name=qtp153449517-9564, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023)
at java.base@16/java.lang.Thread.run(Thread.java:831)
47) Thread[id=9458, name=qtp1312166695-9458, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
48) Thread[id=9490, name=qtp153449517-9490, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023)
at java.base@16/java.lang.Thread.run(Thread.java:831)
49) Thread[id=9514, name=SolrRrdBackendFactory-6588-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base@16/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
at java.base@16/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
at java.base@16/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
at java.base@16/java.lang.Thread.run(Thread.java:831)
50) Thread[id=9675, name=qtp1931265301-9675, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023)
at java.base@16/java.lang.Thread.run(Thread.java:831)
51) Thread[id=9543, name=Thread-1812, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/java.lang.Object.wait(Native Method)
at java.base@16/java.lang.Object.wait(Object.java:320)
at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2281)
52) Thread[id=9481, name=qtp153449517-9481, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
53) Thread[id=9488, name=qtp153449517-9488, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
54) Thread[id=9461, name=qtp1312166695-9461, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/jdk.internal.misc.Unsafe.park(Native Method)
at java.base@16/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
at java.base@16/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:973)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1023)
at java.base@16/java.lang.Thread.run(Thread.java:831)
55) Thread[id=9457, name=qtp1312166695-9457, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderWithPullReplicasTest]
at java.base@16/sun.nio.ch.EPoll.wait(Native Method)
at java.base@16/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
at java.base@16/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
at java.base@16/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183)
at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:360)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:184)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:171)
at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:135)
at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x0000000800f82fb8.run(Unknown Source)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:882)
at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1036)
at java.base@16/java.lang.Thread.run(Thread.java:831)
at __randomizedtesting.SeedInfo.seed([940F3B336DBB6C96]:0)
FAILED: org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest.test
Error Message:
Number of replicas in the state does not match what we set:1 vs 3
Stack Trace:
org.apache.solr.cloud.ZkController$NotInClusterStateException: Number of replicas in the state does not match what we set:1 vs 3
at __randomizedtesting.SeedInfo.seed([940F3B336DBB6C96:1C5B04E9C347016E]:0)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForActiveReplicaCount(AbstractFullDistribZkTestBase.java:635)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:574)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:360)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1088)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1061)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.base/java.lang.Thread.run(Thread.java:831)
FAILED: org.apache.solr.metrics.reporters.solr.SolrCloudReportersTest.testDefaultPlugins
Error Message:
Timeout occurred while waiting response from server at: https://127.0.0.1:43893/solr
Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occurred while waiting response from server at: https://127.0.0.1:43893/solr
at __randomizedtesting.SeedInfo.seed([940F3B336DBB6C96:DC8F69E331FB4FBB]:0)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:692)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:266)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
at org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:369)
at org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:297)
at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.sendRequest(BaseCloudSolrClient.java:1171)
at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.requestWithRetryOnStaleState(BaseCloudSolrClient.java:934)
at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.request(BaseCloudSolrClient.java:866)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:214)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:231)
at org.apache.solr.metrics.reporters.solr.SolrCloudReportersTest.testDefaultPlugins(SolrCloudReportersTest.java:176)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:78)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:567)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.base/java.lang.Thread.run(Thread.java:831)
Caused by: java.net.SocketTimeoutException: Read timed out
at java.base/sun.nio.ch.NioSocketImpl.timedRead(NioSocketImpl.java:283)
at java.base/sun.nio.ch.NioSocketImpl.implRead(NioSocketImpl.java:309)
at java.base/sun.nio.ch.NioSocketImpl.read(NioSocketImpl.java:350)
at java.base/sun.nio.ch.NioSocketImpl$1.read(NioSocketImpl.java:803)
at java.base/java.net.Socket$SocketInputStream.read(Socket.java:976)
at java.base/sun.security.ssl.SSLSocketInputRecord.read(SSLSocketInputRecord.java:478)
at java.base/sun.security.ssl.SSLSocketInputRecord.readHeader(SSLSocketInputRecord.java:472)
at java.base/sun.security.ssl.SSLSocketInputRecord.bytesInCompletePacket(SSLSocketInputRecord.java:70)
at java.base/sun.security.ssl.SSLSocketImpl.readApplicationRecord(SSLSocketImpl.java:1443)
at java.base/sun.security.ssl.SSLSocketImpl$AppInputStream.read(SSLSocketImpl.java:1047)
at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:280)
at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:157)
at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186)
at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:571)
... 52 more