You are viewing a plain text version of this content. The canonical link for it is here.
Posted to builds@solr.apache.org by Policeman Jenkins Server via builds <bu...@solr.apache.org> on 2026/06/14 17:47:58 UTC
[JENKINS] Solr-main-Linux (64bit/hotspot/jdk-22.0.2) - Build # 33879 - Still Unstable!
Build: https://jenkins.thetaphi.de/job/Solr-main-Linux/33879/
Java: 64bit/hotspot/jdk-22.0.2 -XX:-UseCompressedOops -XX:+UseG1GC
1 tests failed.
FAILED: org.apache.solr.opentelemetry.TestDistributedTracing.testV2Api
Error Message:
org.junit.ComparisonFailure: expected:<[post:/collections/{collection}/reloa]d> but was:<[ReloadCollectionCm]d>
Stack Trace:
org.junit.ComparisonFailure: expected:<[post:/collections/{collection}/reloa]d> but was:<[ReloadCollectionCm]d>
at __randomizedtesting.SeedInfo.seed([3B692829D113BC4A:7B8FA1F0412387A3]:0)
at org.junit.Assert.assertEquals(Assert.java:117)
at org.junit.Assert.assertEquals(Assert.java:146)
at org.apache.solr.opentelemetry.TestDistributedTracing.testV2Api(TestDistributedTracing.java:161)
at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103)
at java.base/java.lang.reflect.Method.invoke(Method.java:580)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1763)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48)
at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
at org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
at org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
at java.base/java.lang.Thread.run(Thread.java:1570)
[JENKINS] Solr-main-Linux (64bit/hotspot/jdk-21.0.6) - Build # 33882 - Still Unstable!
Posted by Policeman Jenkins Server via builds <bu...@solr.apache.org>.
Build: https://jenkins.thetaphi.de/job/Solr-main-Linux/33882/
Java: 64bit/hotspot/jdk-21.0.6 -XX:-UseCompressedOops -XX:+UseParallelGC
2 tests failed.
FAILED: org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test
Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occurred when talking to server at: http://127.0.0.1:40817/solr/collection1_shard1_replica_n3/update
Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occurred when talking to server at: http://127.0.0.1:40817/solr/collection1_shard1_replica_n3/update
at __randomizedtesting.SeedInfo.seed([F6694993E1C1373D:7E3D76494F3D5AC5]:0)
at app//org.apache.solr.client.solrj.jetty.HttpJettySolrClient.request(HttpJettySolrClient.java:507)
at app//org.apache.solr.client.solrj.jetty.HttpJettySolrClient.lambda$requestWithBaseUrl$0(HttpJettySolrClient.java:529)
at app//org.apache.solr.client.solrj.jetty.HttpJettySolrClient.requestWithBaseUrl(HttpJettySolrClient.java:548)
at app//org.apache.solr.client.solrj.jetty.HttpJettySolrClient.requestWithBaseUrl(HttpJettySolrClient.java:529)
at app//org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:625)
at app//org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:613)
at app//org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:638)
at app//org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:582)
at app//org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1564)
at app//org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1199)
at app//org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:1121)
at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:302)
at app//org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:167)
at app//org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:132)
at app//org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:148)
at app//org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:106)
at java.base@21.0.6/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103)
at java.base@21.0.6/java.lang.reflect.Method.invoke(Method.java:580)
at app//com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1763)
at app//com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
at app//com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
at app//com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1071)
at app//org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1042)
at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
at app//org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48)
at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
at app//org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
at app//org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
at app//org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at app//org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at app//org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
at app//org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
at app//org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
at app//org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
at java.base@21.0.6/java.lang.Thread.run(Thread.java:1583)
Caused by: java.nio.channels.ClosedChannelException
at org.eclipse.jetty.http2.HTTP2Session$StreamsState.onShutdown(HTTP2Session.java:2141)
at org.eclipse.jetty.http2.HTTP2Session.onShutdown(HTTP2Session.java:1179)
at org.eclipse.jetty.http2.HTTP2Connection$HTTP2Producer.produce(HTTP2Connection.java:389)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produceTask(AdaptiveExecutionStrategy.java:526)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:251)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produce(AdaptiveExecutionStrategy.java:196)
at org.eclipse.jetty.http2.HTTP2Connection.produce(HTTP2Connection.java:216)
at org.eclipse.jetty.http2.HTTP2Connection.onFillable(HTTP2Connection.java:157)
at org.eclipse.jetty.http2.HTTP2Connection$FillableCallback.succeeded(HTTP2Connection.java:468)
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:105)
at org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53)
at org.eclipse.jetty.util.thread.Invocable.invokeNonBlocking(Invocable.java:224)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.picRunTask(AdaptiveExecutionStrategy.java:473)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.consumeTask(AdaptiveExecutionStrategy.java:399)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:255)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produce(AdaptiveExecutionStrategy.java:196)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:408)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
... 1 more
FAILED: org.apache.solr.opentelemetry.TestDistributedTracing.testV2Api
Error Message:
org.junit.ComparisonFailure: expected:<[post:/collections/{collection}/reloa]d> but was:<[ReloadCollectionCm]d>
Stack Trace:
org.junit.ComparisonFailure: expected:<[post:/collections/{collection}/reloa]d> but was:<[ReloadCollectionCm]d>
at __randomizedtesting.SeedInfo.seed([F6694993E1C1373D:B68FC04A71F10CD4]:0)
at org.junit.Assert.assertEquals(Assert.java:117)
at org.junit.Assert.assertEquals(Assert.java:146)
at org.apache.solr.opentelemetry.TestDistributedTracing.testV2Api(TestDistributedTracing.java:161)
at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103)
at java.base/java.lang.reflect.Method.invoke(Method.java:580)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1763)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48)
at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
at org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
at org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
at java.base/java.lang.Thread.run(Thread.java:1583)
[JENKINS] Solr-main-Linux (64bit/hotspot/jdk-22.0.2) - Build # 33881 - Still Unstable!
Posted by Policeman Jenkins Server via builds <bu...@solr.apache.org>.
Build: https://jenkins.thetaphi.de/job/Solr-main-Linux/33881/
Java: 64bit/hotspot/jdk-22.0.2 -XX:-UseCompressedOops -XX:+UseZGC
4 tests failed.
FAILED: org.apache.solr.crossdc.manager.SolrAndKafkaReindexTest.classMethod
Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 75 threads leaked from SUITE scope at org.apache.solr.crossdc.manager.SolrAndKafkaReindexTest:
1) Thread[id=78, name=data-plane-kafka-network-thread-0-ListenerName(PLAINTEXT)-PLAINTEXT-1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at kafka.network.Processor.poll(SocketServer.scala:1110)
at kafka.network.Processor.run(SocketServer.scala:1014)
at java.base/java.lang.Thread.run(Thread.java:1570)
2) Thread[id=536, name=h2sc-253-thread-8, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
3) Thread[id=79, name=data-plane-kafka-network-thread-0-ListenerName(PLAINTEXT)-PLAINTEXT-2, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at kafka.network.Processor.poll(SocketServer.scala:1110)
at kafka.network.Processor.run(SocketServer.scala:1014)
at java.base/java.lang.Thread.run(Thread.java:1570)
4) Thread[id=749, name=zkCallback-255-thread-4, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:410)
at java.base/java.util.concurrent.LinkedTransferQueue$DualNode.await(LinkedTransferQueue.java:452)
at java.base/java.util.concurrent.SynchronousQueue$Transferer.xferLifo(SynchronousQueue.java:194)
at java.base/java.util.concurrent.SynchronousQueue.xfer(SynchronousQueue.java:233)
at java.base/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:336)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1069)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
5) Thread[id=67, name=ThrottledChannelReaper-ControllerMutation, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:291)
at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:178)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
6) Thread[id=64, name=ThrottledChannelReaper-Fetch, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:291)
at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:178)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
7) Thread[id=537, name=h2sc-253-thread-9, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
8) Thread[id=521, name=Curator-ConnectionStateManager-0, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:435)
at org.apache.curator.framework.state.ConnectionStateManager.processEvents(ConnectionStateManager.java:246)
at org.apache.curator.framework.state.ConnectionStateManager.lambda$start$0(ConnectionStateManager.java:127)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
9) Thread[id=474, name=CrossDCConsumer-245-thread-1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:596)
at org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.poll(ConsumerNetworkClient.java:282)
at org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.poll(ConsumerNetworkClient.java:253)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:692)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.solr.crossdc.manager.consumer.KafkaCrossDcConsumer.pollAndProcessRequests(KafkaCrossDcConsumer.java:372)
at org.apache.solr.crossdc.manager.consumer.KafkaCrossDcConsumer.run(KafkaCrossDcConsumer.java:291)
at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:408)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
10) Thread[id=92, name=controller-event-thread, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
at java.base/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
at kafka.controller.ControllerEventManager.kafka$controller$ControllerEventManager$$pollFromEventQueue(ControllerEventManager.scala:156)
at kafka.controller.ControllerEventManager$ControllerEventThread.doWork(ControllerEventManager.scala:129)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
11) Thread[id=63, name=SensorExpiryThread, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
12) Thread[id=101, name=data-plane-kafka-request-handler-0, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:435)
at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:484)
at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:113)
at java.base/java.lang.Thread.run(Thread.java:1570)
13) Thread[id=59, name=SUITE-SolrAndKafkaReindexTest-seed#[CEE9CF5DFD9E00B9]-worker-EventThread, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:537)
14) Thread[id=542, name=h2sc-253-thread-13, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
15) Thread[id=524, name=Curator-Framework-0, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at java.base/java.util.concurrent.DelayQueue.take(DelayQueue.java:242)
at java.base/java.util.concurrent.DelayQueue.take(DelayQueue.java:100)
at org.apache.curator.framework.imps.CuratorFrameworkImpl.backgroundOperationsLoop(CuratorFrameworkImpl.java:796)
at org.apache.curator.framework.imps.CuratorFrameworkImpl.access$300(CuratorFrameworkImpl.java:77)
at org.apache.curator.framework.imps.CuratorFrameworkImpl$4.call(CuratorFrameworkImpl.java:320)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
16) Thread[id=77, name=data-plane-kafka-network-thread-0-ListenerName(PLAINTEXT)-PLAINTEXT-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at kafka.network.Processor.poll(SocketServer.scala:1110)
at kafka.network.Processor.run(SocketServer.scala:1014)
at java.base/java.lang.Thread.run(Thread.java:1570)
17) Thread[id=62, name=NIOWorkerThread-3, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
18) Thread[id=527, name=HttpClient@79e42938-scheduler-1, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
19) Thread[id=95, name=ExpirationReaper-0-Rebalance, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:291)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
20) Thread[id=520, name=h2sc-253-thread-2-processing-collection1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:182)
at org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:189)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:616)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:550)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produceTask(AdaptiveExecutionStrategy.java:526)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:251)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produce(AdaptiveExecutionStrategy.java:196)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:408)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
21) Thread[id=106, name=data-plane-kafka-request-handler-5, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:435)
at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:484)
at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:113)
at java.base/java.lang.Thread.run(Thread.java:1570)
22) Thread[id=709, name=h2sc-253-thread-18, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
23) Thread[id=84, name=ExpirationReaper-0-DeleteRecords, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:291)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
24) Thread[id=58, name=SUITE-SolrAndKafkaReindexTest-seed#[CEE9CF5DFD9E00B9]-worker-SendThread(127.0.0.1:33335), state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1290)
25) Thread[id=60, name=NIOWorkerThread-1, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
26) Thread[id=82, name=ExpirationReaper-0-Produce, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:291)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
27) Thread[id=530, name=h2sc-253-thread-5, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
28) Thread[id=47, name=NIOServerCxnFactory.SelectorThread-1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.select(NIOServerCnxnFactory.java:403)
at org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.run(NIOServerCnxnFactory.java:368)
29) Thread[id=519, name=h2sc-253-thread-1-processing-collection1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:182)
at org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:189)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:616)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:550)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produceTask(AdaptiveExecutionStrategy.java:526)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:251)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produce(AdaptiveExecutionStrategy.java:196)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:408)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
30) Thread[id=103, name=data-plane-kafka-request-handler-2, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:435)
at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:484)
at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:113)
at java.base/java.lang.Thread.run(Thread.java:1570)
31) Thread[id=528, name=h2sc-253-thread-3, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
32) Thread[id=99, name=Controller-0-to-broker-0-send-thread, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:242)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
33) Thread[id=61, name=NIOWorkerThread-2, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
34) Thread[id=517, name=executor-Rebalance, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
35) Thread[id=46, name=NIOServerCxnFactory.SelectorThread-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.select(NIOServerCnxnFactory.java:403)
at org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.run(NIOServerCnxnFactory.java:368)
36) Thread[id=109, name=/config/changes-event-process-thread, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at kafka.common.ZkNodeChangeNotificationListener$ChangeEventProcessThread.doWork(ZkNodeChangeNotificationListener.scala:147)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
37) Thread[id=93, name=ExpirationReaper-0-topic, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:291)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
38) Thread[id=107, name=data-plane-kafka-request-handler-6, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:435)
at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:484)
at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:113)
at java.base/java.lang.Thread.run(Thread.java:1570)
39) Thread[id=94, name=ExpirationReaper-0-Heartbeat, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:300)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
40) Thread[id=50, name=SyncThread:0, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at org.apache.zookeeper.server.SyncRequestProcessor.run(SyncRequestProcessor.java:170)
41) Thread[id=52, name=RequestThrottler, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at org.apache.zookeeper.server.RequestThrottler.run(RequestThrottler.java:147)
42) Thread[id=706, name=h2sc-253-thread-15, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
43) Thread[id=91, name=LogDirFailureHandler, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at java.base/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:420)
at org.apache.kafka.storage.internals.log.LogDirFailureChannel.takeNextOfflineLogDir(LogDirFailureChannel.java:75)
at kafka.server.ReplicaManager$LogDirFailureHandler.doWork(ReplicaManager.scala:320)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
44) Thread[id=81, name=AddPartitionsToTxnSenderThread-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:596)
at org.apache.kafka.server.util.InterBrokerSendThread.pollOnce(InterBrokerSendThread.java:110)
at org.apache.kafka.server.util.InterBrokerSendThread.doWork(InterBrokerSendThread.java:137)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
45) Thread[id=86, name=ExpirationReaper-0-RemoteFetch, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:291)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
46) Thread[id=104, name=data-plane-kafka-request-handler-3, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:435)
at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:484)
at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:113)
at java.base/java.lang.Thread.run(Thread.java:1570)
47) Thread[id=97, name=TxnMarkerSenderThread-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:596)
at org.apache.kafka.server.util.InterBrokerSendThread.pollOnce(InterBrokerSendThread.java:110)
at org.apache.kafka.server.util.InterBrokerSendThread.doWork(InterBrokerSendThread.java:137)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
48) Thread[id=76, name=data-plane-kafka-socket-acceptor-ListenerName(PLAINTEXT)-PLAINTEXT-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at kafka.network.Acceptor.acceptNewConnections(SocketServer.scala:745)
at kafka.network.Acceptor.run(SocketServer.scala:693)
at java.base/java.lang.Thread.run(Thread.java:1570)
49) Thread[id=98, name=transaction-log-manager-0, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
50) Thread[id=747, name=zkCallback-255-thread-2, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:410)
at java.base/java.util.concurrent.LinkedTransferQueue$DualNode.await(LinkedTransferQueue.java:452)
at java.base/java.util.concurrent.SynchronousQueue$Transferer.xferLifo(SynchronousQueue.java:194)
at java.base/java.util.concurrent.SynchronousQueue.xfer(SynchronousQueue.java:233)
at java.base/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:336)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1069)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
51) Thread[id=108, name=data-plane-kafka-request-handler-7, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:435)
at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:484)
at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:113)
at java.base/java.lang.Thread.run(Thread.java:1570)
52) Thread[id=538, name=h2sc-253-thread-10, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
53) Thread[id=707, name=h2sc-253-thread-16, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
54) Thread[id=708, name=h2sc-253-thread-17, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
55) Thread[id=51, name=ProcessThread(sid:0 cport:33335):, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:142)
56) Thread[id=529, name=h2sc-253-thread-4, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
57) Thread[id=540, name=h2sc-253-thread-12, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
58) Thread[id=85, name=ExpirationReaper-0-ElectLeader, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:291)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
59) Thread[id=96, name=group-metadata-manager-0, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
60) Thread[id=75, name=zk-broker-0-to-controller-forwarding-channel-manager, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:596)
at org.apache.kafka.server.util.InterBrokerSendThread.pollOnce(InterBrokerSendThread.java:110)
at kafka.server.NodeToControllerRequestThread.doWork(NodeToControllerChannelManager.scala:377)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
61) Thread[id=627, name=executor-Fetch, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
62) Thread[id=105, name=data-plane-kafka-request-handler-4, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:435)
at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:484)
at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:113)
at java.base/java.lang.Thread.run(Thread.java:1570)
63) Thread[id=83, name=ExpirationReaper-0-Fetch, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:300)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
64) Thread[id=102, name=data-plane-kafka-request-handler-1, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:435)
at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:484)
at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:113)
at java.base/java.lang.Thread.run(Thread.java:1570)
65) Thread[id=48, name=NIOServerCxnFactory.AcceptThread:/127.0.0.1:0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.apache.zookeeper.server.NIOServerCnxnFactory$AcceptThread.select(NIOServerCnxnFactory.java:205)
at org.apache.zookeeper.server.NIOServerCnxnFactory$AcceptThread.run(NIOServerCnxnFactory.java:181)
66) Thread[id=80, name=zk-broker-0-to-controller-alter-partition-channel-manager, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:596)
at org.apache.kafka.server.util.InterBrokerSendThread.pollOnce(InterBrokerSendThread.java:110)
at kafka.server.NodeToControllerRequestThread.doWork(NodeToControllerChannelManager.scala:377)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
67) Thread[id=539, name=h2sc-253-thread-11, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
68) Thread[id=66, name=ThrottledChannelReaper-Request, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:291)
at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:178)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
69) Thread[id=532, name=h2sc-253-thread-6, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
70) Thread[id=100, name=ExpirationReaper-0-AlterAcls, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:291)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
71) Thread[id=748, name=zkCallback-255-thread-3, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:410)
at java.base/java.util.concurrent.LinkedTransferQueue$DualNode.await(LinkedTransferQueue.java:452)
at java.base/java.util.concurrent.SynchronousQueue$Transferer.xferLifo(SynchronousQueue.java:194)
at java.base/java.util.concurrent.SynchronousQueue.xfer(SynchronousQueue.java:233)
at java.base/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:336)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1069)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
72) Thread[id=74, name=feature-zk-node-event-process-thread, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at kafka.server.FinalizedFeatureChangeListener$ChangeNotificationProcessorThread.doWork(FinalizedFeatureChangeListener.scala:154)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
73) Thread[id=65, name=ThrottledChannelReaper-Produce, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:291)
at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:178)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
74) Thread[id=544, name=h2sc-253-thread-14, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
75) Thread[id=535, name=h2sc-253-thread-7, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 75 threads leaked from SUITE scope at org.apache.solr.crossdc.manager.SolrAndKafkaReindexTest:
1) Thread[id=78, name=data-plane-kafka-network-thread-0-ListenerName(PLAINTEXT)-PLAINTEXT-1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at kafka.network.Processor.poll(SocketServer.scala:1110)
at kafka.network.Processor.run(SocketServer.scala:1014)
at java.base/java.lang.Thread.run(Thread.java:1570)
2) Thread[id=536, name=h2sc-253-thread-8, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
3) Thread[id=79, name=data-plane-kafka-network-thread-0-ListenerName(PLAINTEXT)-PLAINTEXT-2, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at kafka.network.Processor.poll(SocketServer.scala:1110)
at kafka.network.Processor.run(SocketServer.scala:1014)
at java.base/java.lang.Thread.run(Thread.java:1570)
4) Thread[id=749, name=zkCallback-255-thread-4, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:410)
at java.base/java.util.concurrent.LinkedTransferQueue$DualNode.await(LinkedTransferQueue.java:452)
at java.base/java.util.concurrent.SynchronousQueue$Transferer.xferLifo(SynchronousQueue.java:194)
at java.base/java.util.concurrent.SynchronousQueue.xfer(SynchronousQueue.java:233)
at java.base/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:336)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1069)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
5) Thread[id=67, name=ThrottledChannelReaper-ControllerMutation, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:291)
at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:178)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
6) Thread[id=64, name=ThrottledChannelReaper-Fetch, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:291)
at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:178)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
7) Thread[id=537, name=h2sc-253-thread-9, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
8) Thread[id=521, name=Curator-ConnectionStateManager-0, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:435)
at org.apache.curator.framework.state.ConnectionStateManager.processEvents(ConnectionStateManager.java:246)
at org.apache.curator.framework.state.ConnectionStateManager.lambda$start$0(ConnectionStateManager.java:127)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
9) Thread[id=474, name=CrossDCConsumer-245-thread-1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:596)
at org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.poll(ConsumerNetworkClient.java:282)
at org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.poll(ConsumerNetworkClient.java:253)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:692)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.solr.crossdc.manager.consumer.KafkaCrossDcConsumer.pollAndProcessRequests(KafkaCrossDcConsumer.java:372)
at org.apache.solr.crossdc.manager.consumer.KafkaCrossDcConsumer.run(KafkaCrossDcConsumer.java:291)
at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:408)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
10) Thread[id=92, name=controller-event-thread, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
at java.base/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
at kafka.controller.ControllerEventManager.kafka$controller$ControllerEventManager$$pollFromEventQueue(ControllerEventManager.scala:156)
at kafka.controller.ControllerEventManager$ControllerEventThread.doWork(ControllerEventManager.scala:129)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
11) Thread[id=63, name=SensorExpiryThread, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
12) Thread[id=101, name=data-plane-kafka-request-handler-0, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:435)
at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:484)
at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:113)
at java.base/java.lang.Thread.run(Thread.java:1570)
13) Thread[id=59, name=SUITE-SolrAndKafkaReindexTest-seed#[CEE9CF5DFD9E00B9]-worker-EventThread, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:537)
14) Thread[id=542, name=h2sc-253-thread-13, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
15) Thread[id=524, name=Curator-Framework-0, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at java.base/java.util.concurrent.DelayQueue.take(DelayQueue.java:242)
at java.base/java.util.concurrent.DelayQueue.take(DelayQueue.java:100)
at org.apache.curator.framework.imps.CuratorFrameworkImpl.backgroundOperationsLoop(CuratorFrameworkImpl.java:796)
at org.apache.curator.framework.imps.CuratorFrameworkImpl.access$300(CuratorFrameworkImpl.java:77)
at org.apache.curator.framework.imps.CuratorFrameworkImpl$4.call(CuratorFrameworkImpl.java:320)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
16) Thread[id=77, name=data-plane-kafka-network-thread-0-ListenerName(PLAINTEXT)-PLAINTEXT-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at kafka.network.Processor.poll(SocketServer.scala:1110)
at kafka.network.Processor.run(SocketServer.scala:1014)
at java.base/java.lang.Thread.run(Thread.java:1570)
17) Thread[id=62, name=NIOWorkerThread-3, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
18) Thread[id=527, name=HttpClient@79e42938-scheduler-1, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
19) Thread[id=95, name=ExpirationReaper-0-Rebalance, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:291)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
20) Thread[id=520, name=h2sc-253-thread-2-processing-collection1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:182)
at org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:189)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:616)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:550)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produceTask(AdaptiveExecutionStrategy.java:526)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:251)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produce(AdaptiveExecutionStrategy.java:196)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:408)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
21) Thread[id=106, name=data-plane-kafka-request-handler-5, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:435)
at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:484)
at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:113)
at java.base/java.lang.Thread.run(Thread.java:1570)
22) Thread[id=709, name=h2sc-253-thread-18, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
23) Thread[id=84, name=ExpirationReaper-0-DeleteRecords, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:291)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
24) Thread[id=58, name=SUITE-SolrAndKafkaReindexTest-seed#[CEE9CF5DFD9E00B9]-worker-SendThread(127.0.0.1:33335), state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1290)
25) Thread[id=60, name=NIOWorkerThread-1, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
26) Thread[id=82, name=ExpirationReaper-0-Produce, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:291)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
27) Thread[id=530, name=h2sc-253-thread-5, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
28) Thread[id=47, name=NIOServerCxnFactory.SelectorThread-1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.select(NIOServerCnxnFactory.java:403)
at org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.run(NIOServerCnxnFactory.java:368)
29) Thread[id=519, name=h2sc-253-thread-1-processing-collection1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:182)
at org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:189)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:616)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:550)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produceTask(AdaptiveExecutionStrategy.java:526)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:251)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produce(AdaptiveExecutionStrategy.java:196)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:408)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
30) Thread[id=103, name=data-plane-kafka-request-handler-2, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:435)
at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:484)
at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:113)
at java.base/java.lang.Thread.run(Thread.java:1570)
31) Thread[id=528, name=h2sc-253-thread-3, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
32) Thread[id=99, name=Controller-0-to-broker-0-send-thread, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:242)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
33) Thread[id=61, name=NIOWorkerThread-2, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
34) Thread[id=517, name=executor-Rebalance, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
35) Thread[id=46, name=NIOServerCxnFactory.SelectorThread-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.select(NIOServerCnxnFactory.java:403)
at org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.run(NIOServerCnxnFactory.java:368)
36) Thread[id=109, name=/config/changes-event-process-thread, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at kafka.common.ZkNodeChangeNotificationListener$ChangeEventProcessThread.doWork(ZkNodeChangeNotificationListener.scala:147)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
37) Thread[id=93, name=ExpirationReaper-0-topic, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:291)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
38) Thread[id=107, name=data-plane-kafka-request-handler-6, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:435)
at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:484)
at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:113)
at java.base/java.lang.Thread.run(Thread.java:1570)
39) Thread[id=94, name=ExpirationReaper-0-Heartbeat, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:300)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
40) Thread[id=50, name=SyncThread:0, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at org.apache.zookeeper.server.SyncRequestProcessor.run(SyncRequestProcessor.java:170)
41) Thread[id=52, name=RequestThrottler, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at org.apache.zookeeper.server.RequestThrottler.run(RequestThrottler.java:147)
42) Thread[id=706, name=h2sc-253-thread-15, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
43) Thread[id=91, name=LogDirFailureHandler, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at java.base/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:420)
at org.apache.kafka.storage.internals.log.LogDirFailureChannel.takeNextOfflineLogDir(LogDirFailureChannel.java:75)
at kafka.server.ReplicaManager$LogDirFailureHandler.doWork(ReplicaManager.scala:320)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
44) Thread[id=81, name=AddPartitionsToTxnSenderThread-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:596)
at org.apache.kafka.server.util.InterBrokerSendThread.pollOnce(InterBrokerSendThread.java:110)
at org.apache.kafka.server.util.InterBrokerSendThread.doWork(InterBrokerSendThread.java:137)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
45) Thread[id=86, name=ExpirationReaper-0-RemoteFetch, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:291)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
46) Thread[id=104, name=data-plane-kafka-request-handler-3, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:435)
at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:484)
at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:113)
at java.base/java.lang.Thread.run(Thread.java:1570)
47) Thread[id=97, name=TxnMarkerSenderThread-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:596)
at org.apache.kafka.server.util.InterBrokerSendThread.pollOnce(InterBrokerSendThread.java:110)
at org.apache.kafka.server.util.InterBrokerSendThread.doWork(InterBrokerSendThread.java:137)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
48) Thread[id=76, name=data-plane-kafka-socket-acceptor-ListenerName(PLAINTEXT)-PLAINTEXT-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at kafka.network.Acceptor.acceptNewConnections(SocketServer.scala:745)
at kafka.network.Acceptor.run(SocketServer.scala:693)
at java.base/java.lang.Thread.run(Thread.java:1570)
49) Thread[id=98, name=transaction-log-manager-0, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
50) Thread[id=747, name=zkCallback-255-thread-2, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:410)
at java.base/java.util.concurrent.LinkedTransferQueue$DualNode.await(LinkedTransferQueue.java:452)
at java.base/java.util.concurrent.SynchronousQueue$Transferer.xferLifo(SynchronousQueue.java:194)
at java.base/java.util.concurrent.SynchronousQueue.xfer(SynchronousQueue.java:233)
at java.base/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:336)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1069)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
51) Thread[id=108, name=data-plane-kafka-request-handler-7, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:435)
at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:484)
at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:113)
at java.base/java.lang.Thread.run(Thread.java:1570)
52) Thread[id=538, name=h2sc-253-thread-10, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
53) Thread[id=707, name=h2sc-253-thread-16, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
54) Thread[id=708, name=h2sc-253-thread-17, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
55) Thread[id=51, name=ProcessThread(sid:0 cport:33335):, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:142)
56) Thread[id=529, name=h2sc-253-thread-4, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
57) Thread[id=540, name=h2sc-253-thread-12, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
58) Thread[id=85, name=ExpirationReaper-0-ElectLeader, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:291)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
59) Thread[id=96, name=group-metadata-manager-0, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
60) Thread[id=75, name=zk-broker-0-to-controller-forwarding-channel-manager, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:596)
at org.apache.kafka.server.util.InterBrokerSendThread.pollOnce(InterBrokerSendThread.java:110)
at kafka.server.NodeToControllerRequestThread.doWork(NodeToControllerChannelManager.scala:377)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
61) Thread[id=627, name=executor-Fetch, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
62) Thread[id=105, name=data-plane-kafka-request-handler-4, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:435)
at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:484)
at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:113)
at java.base/java.lang.Thread.run(Thread.java:1570)
63) Thread[id=83, name=ExpirationReaper-0-Fetch, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:300)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
64) Thread[id=102, name=data-plane-kafka-request-handler-1, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:435)
at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:484)
at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:113)
at java.base/java.lang.Thread.run(Thread.java:1570)
65) Thread[id=48, name=NIOServerCxnFactory.AcceptThread:/127.0.0.1:0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.apache.zookeeper.server.NIOServerCnxnFactory$AcceptThread.select(NIOServerCnxnFactory.java:205)
at org.apache.zookeeper.server.NIOServerCnxnFactory$AcceptThread.run(NIOServerCnxnFactory.java:181)
66) Thread[id=80, name=zk-broker-0-to-controller-alter-partition-channel-manager, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:596)
at org.apache.kafka.server.util.InterBrokerSendThread.pollOnce(InterBrokerSendThread.java:110)
at kafka.server.NodeToControllerRequestThread.doWork(NodeToControllerChannelManager.scala:377)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
67) Thread[id=539, name=h2sc-253-thread-11, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
68) Thread[id=66, name=ThrottledChannelReaper-Request, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:291)
at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:178)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
69) Thread[id=532, name=h2sc-253-thread-6, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
70) Thread[id=100, name=ExpirationReaper-0-AlterAcls, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:291)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
71) Thread[id=748, name=zkCallback-255-thread-3, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:410)
at java.base/java.util.concurrent.LinkedTransferQueue$DualNode.await(LinkedTransferQueue.java:452)
at java.base/java.util.concurrent.SynchronousQueue$Transferer.xferLifo(SynchronousQueue.java:194)
at java.base/java.util.concurrent.SynchronousQueue.xfer(SynchronousQueue.java:233)
at java.base/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:336)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1069)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
72) Thread[id=74, name=feature-zk-node-event-process-thread, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at kafka.server.FinalizedFeatureChangeListener$ChangeNotificationProcessorThread.doWork(FinalizedFeatureChangeListener.scala:154)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
73) Thread[id=65, name=ThrottledChannelReaper-Produce, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:291)
at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:178)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
74) Thread[id=544, name=h2sc-253-thread-14, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
75) Thread[id=535, name=h2sc-253-thread-7, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
at __randomizedtesting.SeedInfo.seed([CEE9CF5DFD9E00B9]:0)
FAILED: org.apache.solr.crossdc.manager.SolrAndKafkaReindexTest.classMethod
Error Message:
java.lang.Exception: Error shutting down MiniSolrCloudCluster
Stack Trace:
java.lang.Exception: Error shutting down MiniSolrCloudCluster
at __randomizedtesting.SeedInfo.seed([CEE9CF5DFD9E00B9]:0)
at org.apache.solr.cloud.MiniSolrCloudCluster.checkForExceptions(MiniSolrCloudCluster.java:713)
at org.apache.solr.cloud.MiniSolrCloudCluster.shutdown(MiniSolrCloudCluster.java:637)
at org.apache.solr.crossdc.manager.SolrAndKafkaReindexTest.afterSolrAndKafkaIntegrationTest(SolrAndKafkaReindexTest.java:138)
at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103)
at java.base/java.lang.reflect.Method.invoke(Method.java:580)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1763)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:909)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
at org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
at java.base/java.lang.Thread.run(Thread.java:1570)
Suppressed: java.lang.RuntimeException: Timeout waiting for pool to shutdown. [poolName=closeThreadPool][poolName=closeThreadPool] org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor@7f00f161[Shutting down, pool size = 1, active threads = 1, queued tasks = 0, completed tasks = 4]
at org.apache.solr.common.util.ExecutorUtil.awaitTermination(ExecutorUtil.java:167)
at org.apache.solr.common.util.ExecutorUtil.awaitTermination(ExecutorUtil.java:153)
at org.apache.solr.common.util.ExecutorUtil.shutdownAndAwaitTermination(ExecutorUtil.java:119)
at org.apache.solr.core.CoreContainer.shutdown(CoreContainer.java:1325)
at org.apache.solr.servlet.CoreContainerProvider.close(CoreContainerProvider.java:132)
at org.apache.solr.servlet.CoreContainerProvider.contextDestroyed(CoreContainerProvider.java:90)
at org.eclipse.jetty.ee10.servlet.ServletContextHandler.callContextDestroyed(ServletContextHandler.java:1608)
at org.eclipse.jetty.ee10.servlet.ServletContextHandler.contextDestroyed(ServletContextHandler.java:529)
at org.eclipse.jetty.ee10.servlet.ServletHandler.doStop(ServletHandler.java:329)
at org.eclipse.jetty.util.component.AbstractLifeCycle.stop(AbstractLifeCycle.java:132)
at org.eclipse.jetty.util.component.ContainerLifeCycle.stop(ContainerLifeCycle.java:180)
at org.eclipse.jetty.util.component.ContainerLifeCycle.doStop(ContainerLifeCycle.java:202)
at org.eclipse.jetty.server.Handler$Abstract.doStop(Handler.java:499)
at org.eclipse.jetty.server.handler.ContextHandler.lambda$doStop$0(ContextHandler.java:841)
at org.eclipse.jetty.server.handler.ContextHandler$ScopedContext.call(ContextHandler.java:1488)
at org.eclipse.jetty.server.handler.ContextHandler.doStop(ContextHandler.java:841)
at org.eclipse.jetty.ee10.servlet.ServletContextHandler.stopContext(ServletContextHandler.java:1334)
at org.eclipse.jetty.ee10.servlet.ServletContextHandler.doStop(ServletContextHandler.java:1085)
at org.eclipse.jetty.util.component.AbstractLifeCycle.stop(AbstractLifeCycle.java:132)
at org.eclipse.jetty.util.component.ContainerLifeCycle.stop(ContainerLifeCycle.java:180)
at org.eclipse.jetty.util.component.ContainerLifeCycle.doStop(ContainerLifeCycle.java:202)
at org.eclipse.jetty.server.Handler$Abstract.doStop(Handler.java:499)
at org.eclipse.jetty.util.component.AbstractLifeCycle.stop(AbstractLifeCycle.java:132)
at org.eclipse.jetty.util.component.ContainerLifeCycle.stop(ContainerLifeCycle.java:180)
at org.eclipse.jetty.util.component.ContainerLifeCycle.doStop(ContainerLifeCycle.java:202)
at org.eclipse.jetty.server.Handler$Abstract.doStop(Handler.java:499)
at org.eclipse.jetty.server.Server.doStop(Server.java:690)
at org.eclipse.jetty.util.component.AbstractLifeCycle.stop(AbstractLifeCycle.java:132)
at org.apache.solr.embedded.JettySolrRunner.stop(JettySolrRunner.java:590)
at org.apache.solr.cloud.MiniSolrCloudCluster.stopJettySolrRunner(MiniSolrCloudCluster.java:529)
at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$shutdown$5(MiniSolrCloudCluster.java:629)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:408)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
... 1 more
FAILED: org.apache.solr.crossdc.manager.SolrAndKafkaReindexTest.classMethod
Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=78, name=data-plane-kafka-network-thread-0-ListenerName(PLAINTEXT)-PLAINTEXT-1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.IOUtil.drain(Native Method)
at java.base/sun.nio.ch.EventFD.reset(EventFD.java:52)
at java.base/sun.nio.ch.EPollSelectorImpl.clearInterrupt(EPollSelectorImpl.java:269)
at java.base/sun.nio.ch.EPollSelectorImpl.processEvents(EPollSelectorImpl.java:207)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:141)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at kafka.network.Processor.poll(SocketServer.scala:1110)
at kafka.network.Processor.run(SocketServer.scala:1014)
at java.base/java.lang.Thread.run(Thread.java:1570)
2) Thread[id=706, name=h2sc-253-thread-15, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
3) Thread[id=536, name=h2sc-253-thread-8, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
4) Thread[id=79, name=data-plane-kafka-network-thread-0-ListenerName(PLAINTEXT)-PLAINTEXT-2, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/java.lang.Thread.holdsLock(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:104)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at kafka.network.Processor.poll(SocketServer.scala:1110)
at kafka.network.Processor.run(SocketServer.scala:1014)
at java.base/java.lang.Thread.run(Thread.java:1570)
5) Thread[id=749, name=zkCallback-255-thread-4, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:410)
at java.base/java.util.concurrent.LinkedTransferQueue$DualNode.await(LinkedTransferQueue.java:452)
at java.base/java.util.concurrent.SynchronousQueue$Transferer.xferLifo(SynchronousQueue.java:194)
at java.base/java.util.concurrent.SynchronousQueue.xfer(SynchronousQueue.java:233)
at java.base/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:336)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1069)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
6) Thread[id=537, name=h2sc-253-thread-9, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
7) Thread[id=521, name=Curator-ConnectionStateManager-0, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:435)
at org.apache.curator.framework.state.ConnectionStateManager.processEvents(ConnectionStateManager.java:246)
at org.apache.curator.framework.state.ConnectionStateManager.lambda$start$0(ConnectionStateManager.java:127)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
8) Thread[id=474, name=CrossDCConsumer-245-thread-1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.unpark(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.unpark(LockSupport.java:181)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer.signalNext(AbstractQueuedSynchronizer.java:645)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer.release(AbstractQueuedSynchronizer.java:1060)
at java.base/java.util.concurrent.locks.ReentrantLock.unlock(ReentrantLock.java:494)
at org.gradle.internal.remote.internal.hub.MessageHub$ChannelDispatch.dispatch(MessageHub.java:375)
at org.gradle.internal.dispatch.ProxyDispatchAdapter$DispatchingInvocationHandler.invoke(ProxyDispatchAdapter.java:92)
at jdk.proxy1/jdk.proxy1.$Proxy5.output(Unknown Source)
at org.gradle.api.internal.tasks.testing.redirector.TestOutputRedirector$Forwarder.onOutput(TestOutputRedirector.java:74)
at org.gradle.api.internal.tasks.testing.redirector.DefaultStandardOutputRedirector$WriteAction.text(DefaultStandardOutputRedirector.java:85)
at org.gradle.internal.io.LineBufferingOutputStream.flush(LineBufferingOutputStream.java:97)
at org.gradle.internal.io.LineBufferingOutputStream.write(LineBufferingOutputStream.java:81)
at java.base/java.io.OutputStream.write(OutputStream.java:167)
at java.base/java.io.PrintStream.implWrite(PrintStream.java:643)
at java.base/java.io.PrintStream.write(PrintStream.java:623)
at org.gradle.internal.io.LinePerThreadBufferingOutputStream.write(LinePerThreadBufferingOutputStream.java:250)
at org.apache.lucene.tests.util.TestRuleLimitSysouts$DelegateStream.write(TestRuleLimitSysouts.java:173)
at java.base/java.io.PrintStream.implWrite(PrintStream.java:643)
at java.base/java.io.PrintStream.write(PrintStream.java:623)
at org.apache.logging.log4j.core.util.CloseShieldOutputStream.write(CloseShieldOutputStream.java:53)
at org.apache.logging.log4j.core.appender.OutputStreamManager.writeToDestination(OutputStreamManager.java:263)
at org.apache.logging.log4j.core.appender.OutputStreamManager.flushBuffer(OutputStreamManager.java:296)
at org.apache.logging.log4j.core.appender.OutputStreamManager.flush(OutputStreamManager.java:307)
at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:229)
at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:220)
at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:211)
at org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:160)
at org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:133)
at org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:124)
at org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:88)
at org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:711)
at org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:669)
at org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:645)
at org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:589)
at org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:92)
at org.apache.logging.log4j.core.Logger.log(Logger.java:187)
at org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2970)
at org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2922)
at org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2904)
at org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2648)
at org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:2587)
at org.apache.logging.slf4j.Log4jLogger.error(Log4jLogger.java:313)
at org.apache.solr.crossdc.manager.consumer.KafkaCrossDcConsumer.pollAndProcessRequests(KafkaCrossDcConsumer.java:563)
at org.apache.solr.crossdc.manager.consumer.KafkaCrossDcConsumer.run(KafkaCrossDcConsumer.java:291)
at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:408)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
9) Thread[id=81, name=AddPartitionsToTxnSenderThread-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.IOUtil.drain(Native Method)
at java.base/sun.nio.ch.EventFD.reset(EventFD.java:52)
at java.base/sun.nio.ch.EPollSelectorImpl.clearInterrupt(EPollSelectorImpl.java:269)
at java.base/sun.nio.ch.EPollSelectorImpl.processEvents(EPollSelectorImpl.java:207)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:141)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:596)
at org.apache.kafka.server.util.InterBrokerSendThread.pollOnce(InterBrokerSendThread.java:110)
at org.apache.kafka.server.util.InterBrokerSendThread.doWork(InterBrokerSendThread.java:137)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
10) Thread[id=63, name=SensorExpiryThread, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
11) Thread[id=542, name=h2sc-253-thread-13, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
12) Thread[id=97, name=TxnMarkerSenderThread-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/java.util.Collections$UnmodifiableCollection.contains(Collections.java:1068)
at java.base/java.util.stream.ReduceOps$3.getOpFlags(ReduceOps.java:185)
at java.base/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:265)
at java.base/java.util.stream.ReferencePipeline.collect(ReferencePipeline.java:702)
at org.apache.kafka.clients.ClusterConnectionStates.nodesWithConnectionSetupTimeout(ClusterConnectionStates.java:460)
at org.apache.kafka.clients.NetworkClient.handleTimedOutConnections(NetworkClient.java:890)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:609)
at org.apache.kafka.server.util.InterBrokerSendThread.pollOnce(InterBrokerSendThread.java:110)
at org.apache.kafka.server.util.InterBrokerSendThread.doWork(InterBrokerSendThread.java:137)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
13) Thread[id=76, name=data-plane-kafka-socket-acceptor-ListenerName(PLAINTEXT)-PLAINTEXT-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EventFD.set0(Native Method)
at java.base/sun.nio.ch.EventFD.set(EventFD.java:48)
at java.base/sun.nio.ch.EPollSelectorImpl.wakeup(EPollSelectorImpl.java:257)
at java.base/java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:231)
at java.base/java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:237)
at java.base/sun.nio.ch.SelectorImpl.begin(SelectorImpl.java:100)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:115)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at kafka.network.Acceptor.acceptNewConnections(SocketServer.scala:745)
at kafka.network.Acceptor.run(SocketServer.scala:693)
at java.base/java.lang.Thread.run(Thread.java:1570)
14) Thread[id=524, name=Curator-Framework-0, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at java.base/java.util.concurrent.DelayQueue.take(DelayQueue.java:242)
at java.base/java.util.concurrent.DelayQueue.take(DelayQueue.java:100)
at org.apache.curator.framework.imps.CuratorFrameworkImpl.backgroundOperationsLoop(CuratorFrameworkImpl.java:796)
at org.apache.curator.framework.imps.CuratorFrameworkImpl.access$300(CuratorFrameworkImpl.java:77)
at org.apache.curator.framework.imps.CuratorFrameworkImpl$4.call(CuratorFrameworkImpl.java:320)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
15) Thread[id=77, name=data-plane-kafka-network-thread-0-ListenerName(PLAINTEXT)-PLAINTEXT-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:112)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at kafka.network.Processor.poll(SocketServer.scala:1110)
at kafka.network.Processor.run(SocketServer.scala:1014)
at java.base/java.lang.Thread.run(Thread.java:1570)
16) Thread[id=98, name=transaction-log-manager-0, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
17) Thread[id=62, name=NIOWorkerThread-3, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
18) Thread[id=747, name=zkCallback-255-thread-2, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:410)
at java.base/java.util.concurrent.LinkedTransferQueue$DualNode.await(LinkedTransferQueue.java:452)
at java.base/java.util.concurrent.SynchronousQueue$Transferer.xferLifo(SynchronousQueue.java:194)
at java.base/java.util.concurrent.SynchronousQueue.xfer(SynchronousQueue.java:233)
at java.base/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:336)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1069)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
19) Thread[id=527, name=HttpClient@79e42938-scheduler-1, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
20) Thread[id=520, name=h2sc-253-thread-2-processing-collection1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:182)
at org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:189)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:616)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:550)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produceTask(AdaptiveExecutionStrategy.java:526)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:251)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produce(AdaptiveExecutionStrategy.java:196)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:408)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
21) Thread[id=709, name=h2sc-253-thread-18, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
22) Thread[id=538, name=h2sc-253-thread-10, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
23) Thread[id=58, name=SUITE-SolrAndKafkaReindexTest-seed#[CEE9CF5DFD9E00B9]-worker-SendThread(127.0.0.1:33335), state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/java.lang.Thread.sleepNanos0(Native Method)
at java.base/java.lang.Thread.sleepNanos(Thread.java:491)
at java.base/java.lang.Thread.sleep(Thread.java:522)
at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:362)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1197)
24) Thread[id=60, name=NIOWorkerThread-1, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
25) Thread[id=707, name=h2sc-253-thread-16, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
26) Thread[id=530, name=h2sc-253-thread-5, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
27) Thread[id=708, name=h2sc-253-thread-17, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
28) Thread[id=529, name=h2sc-253-thread-4, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
29) Thread[id=540, name=h2sc-253-thread-12, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
30) Thread[id=47, name=NIOServerCxnFactory.SelectorThread-1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/java.lang.Thread.holdsLock(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:104)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.select(NIOServerCnxnFactory.java:403)
at org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.run(NIOServerCnxnFactory.java:368)
31) Thread[id=519, name=h2sc-253-thread-1-processing-collection1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:182)
at org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:189)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:616)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:550)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produceTask(AdaptiveExecutionStrategy.java:526)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:251)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produce(AdaptiveExecutionStrategy.java:196)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:408)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
32) Thread[id=96, name=group-metadata-manager-0, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
33) Thread[id=528, name=h2sc-253-thread-3, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
34) Thread[id=75, name=zk-broker-0-to-controller-forwarding-channel-manager, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at org.apache.kafka.common.metrics.Sensor.recordInternal(Sensor.java:232)
at org.apache.kafka.common.metrics.Sensor.record(Sensor.java:226)
at org.apache.kafka.common.network.Selector.poll(Selector.java:487)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:596)
at org.apache.kafka.server.util.InterBrokerSendThread.pollOnce(InterBrokerSendThread.java:110)
at kafka.server.NodeToControllerRequestThread.doWork(NodeToControllerChannelManager.scala:377)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
35) Thread[id=627, name=executor-Fetch, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
36) Thread[id=61, name=NIOWorkerThread-2, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
37) Thread[id=517, name=executor-Rebalance, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
38) Thread[id=48, name=NIOServerCxnFactory.AcceptThread:/127.0.0.1:0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/java.lang.Thread.holdsLock(Native Method)
at java.base/sun.nio.ch.SelectorImpl.processDeregisterQueue(SelectorImpl.java:263)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:140)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.apache.zookeeper.server.NIOServerCnxnFactory$AcceptThread.select(NIOServerCnxnFactory.java:205)
at org.apache.zookeeper.server.NIOServerCnxnFactory$AcceptThread.run(NIOServerCnxnFactory.java:181)
39) Thread[id=80, name=zk-broker-0-to-controller-alter-partition-channel-manager, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/java.util.stream.AbstractPipeline.wrapSink(AbstractPipeline.java:594)
at java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:546)
at java.base/java.util.stream.ReduceOps$ReduceOp.evaluateSequential(ReduceOps.java:921)
at java.base/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:265)
at java.base/java.util.stream.ReferencePipeline.collect(ReferencePipeline.java:702)
at org.apache.kafka.clients.ClusterConnectionStates.nodesWithConnectionSetupTimeout(ClusterConnectionStates.java:460)
at org.apache.kafka.clients.NetworkClient.handleTimedOutConnections(NetworkClient.java:890)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:609)
at org.apache.kafka.server.util.InterBrokerSendThread.pollOnce(InterBrokerSendThread.java:110)
at kafka.server.NodeToControllerRequestThread.doWork(NodeToControllerChannelManager.scala:377)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
40) Thread[id=539, name=h2sc-253-thread-11, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
41) Thread[id=46, name=NIOServerCxnFactory.SelectorThread-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EventFD.set0(Native Method)
at java.base/sun.nio.ch.EventFD.set(EventFD.java:48)
at java.base/sun.nio.ch.EPollSelectorImpl.wakeup(EPollSelectorImpl.java:257)
at java.base/java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:231)
at java.base/java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:237)
at java.base/sun.nio.ch.SelectorImpl.begin(SelectorImpl.java:100)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:115)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.select(NIOServerCnxnFactory.java:403)
at org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.run(NIOServerCnxnFactory.java:368)
42) Thread[id=532, name=h2sc-253-thread-6, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
43) Thread[id=748, name=zkCallback-255-thread-3, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:410)
at java.base/java.util.concurrent.LinkedTransferQueue$DualNode.await(LinkedTransferQueue.java:452)
at java.base/java.util.concurrent.SynchronousQueue$Transferer.xferLifo(SynchronousQueue.java:194)
at java.base/java.util.concurrent.SynchronousQueue.xfer(SynchronousQueue.java:233)
at java.base/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:336)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1069)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
44) Thread[id=544, name=h2sc-253-thread-14, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
45) Thread[id=535, name=h2sc-253-thread-7, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=78, name=data-plane-kafka-network-thread-0-ListenerName(PLAINTEXT)-PLAINTEXT-1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.IOUtil.drain(Native Method)
at java.base/sun.nio.ch.EventFD.reset(EventFD.java:52)
at java.base/sun.nio.ch.EPollSelectorImpl.clearInterrupt(EPollSelectorImpl.java:269)
at java.base/sun.nio.ch.EPollSelectorImpl.processEvents(EPollSelectorImpl.java:207)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:141)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at kafka.network.Processor.poll(SocketServer.scala:1110)
at kafka.network.Processor.run(SocketServer.scala:1014)
at java.base/java.lang.Thread.run(Thread.java:1570)
2) Thread[id=706, name=h2sc-253-thread-15, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
3) Thread[id=536, name=h2sc-253-thread-8, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
4) Thread[id=79, name=data-plane-kafka-network-thread-0-ListenerName(PLAINTEXT)-PLAINTEXT-2, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/java.lang.Thread.holdsLock(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:104)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at kafka.network.Processor.poll(SocketServer.scala:1110)
at kafka.network.Processor.run(SocketServer.scala:1014)
at java.base/java.lang.Thread.run(Thread.java:1570)
5) Thread[id=749, name=zkCallback-255-thread-4, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:410)
at java.base/java.util.concurrent.LinkedTransferQueue$DualNode.await(LinkedTransferQueue.java:452)
at java.base/java.util.concurrent.SynchronousQueue$Transferer.xferLifo(SynchronousQueue.java:194)
at java.base/java.util.concurrent.SynchronousQueue.xfer(SynchronousQueue.java:233)
at java.base/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:336)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1069)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
6) Thread[id=537, name=h2sc-253-thread-9, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
7) Thread[id=521, name=Curator-ConnectionStateManager-0, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:435)
at org.apache.curator.framework.state.ConnectionStateManager.processEvents(ConnectionStateManager.java:246)
at org.apache.curator.framework.state.ConnectionStateManager.lambda$start$0(ConnectionStateManager.java:127)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
8) Thread[id=474, name=CrossDCConsumer-245-thread-1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.unpark(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.unpark(LockSupport.java:181)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer.signalNext(AbstractQueuedSynchronizer.java:645)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer.release(AbstractQueuedSynchronizer.java:1060)
at java.base/java.util.concurrent.locks.ReentrantLock.unlock(ReentrantLock.java:494)
at org.gradle.internal.remote.internal.hub.MessageHub$ChannelDispatch.dispatch(MessageHub.java:375)
at org.gradle.internal.dispatch.ProxyDispatchAdapter$DispatchingInvocationHandler.invoke(ProxyDispatchAdapter.java:92)
at jdk.proxy1/jdk.proxy1.$Proxy5.output(Unknown Source)
at org.gradle.api.internal.tasks.testing.redirector.TestOutputRedirector$Forwarder.onOutput(TestOutputRedirector.java:74)
at org.gradle.api.internal.tasks.testing.redirector.DefaultStandardOutputRedirector$WriteAction.text(DefaultStandardOutputRedirector.java:85)
at org.gradle.internal.io.LineBufferingOutputStream.flush(LineBufferingOutputStream.java:97)
at org.gradle.internal.io.LineBufferingOutputStream.write(LineBufferingOutputStream.java:81)
at java.base/java.io.OutputStream.write(OutputStream.java:167)
at java.base/java.io.PrintStream.implWrite(PrintStream.java:643)
at java.base/java.io.PrintStream.write(PrintStream.java:623)
at org.gradle.internal.io.LinePerThreadBufferingOutputStream.write(LinePerThreadBufferingOutputStream.java:250)
at org.apache.lucene.tests.util.TestRuleLimitSysouts$DelegateStream.write(TestRuleLimitSysouts.java:173)
at java.base/java.io.PrintStream.implWrite(PrintStream.java:643)
at java.base/java.io.PrintStream.write(PrintStream.java:623)
at org.apache.logging.log4j.core.util.CloseShieldOutputStream.write(CloseShieldOutputStream.java:53)
at org.apache.logging.log4j.core.appender.OutputStreamManager.writeToDestination(OutputStreamManager.java:263)
at org.apache.logging.log4j.core.appender.OutputStreamManager.flushBuffer(OutputStreamManager.java:296)
at org.apache.logging.log4j.core.appender.OutputStreamManager.flush(OutputStreamManager.java:307)
at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:229)
at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:220)
at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:211)
at org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:160)
at org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:133)
at org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:124)
at org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:88)
at org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:711)
at org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:669)
at org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:645)
at org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:589)
at org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:92)
at org.apache.logging.log4j.core.Logger.log(Logger.java:187)
at org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2970)
at org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2922)
at org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2904)
at org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2648)
at org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:2587)
at org.apache.logging.slf4j.Log4jLogger.error(Log4jLogger.java:313)
at org.apache.solr.crossdc.manager.consumer.KafkaCrossDcConsumer.pollAndProcessRequests(KafkaCrossDcConsumer.java:563)
at org.apache.solr.crossdc.manager.consumer.KafkaCrossDcConsumer.run(KafkaCrossDcConsumer.java:291)
at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:408)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
9) Thread[id=81, name=AddPartitionsToTxnSenderThread-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.IOUtil.drain(Native Method)
at java.base/sun.nio.ch.EventFD.reset(EventFD.java:52)
at java.base/sun.nio.ch.EPollSelectorImpl.clearInterrupt(EPollSelectorImpl.java:269)
at java.base/sun.nio.ch.EPollSelectorImpl.processEvents(EPollSelectorImpl.java:207)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:141)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:596)
at org.apache.kafka.server.util.InterBrokerSendThread.pollOnce(InterBrokerSendThread.java:110)
at org.apache.kafka.server.util.InterBrokerSendThread.doWork(InterBrokerSendThread.java:137)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
10) Thread[id=63, name=SensorExpiryThread, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
11) Thread[id=542, name=h2sc-253-thread-13, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
12) Thread[id=97, name=TxnMarkerSenderThread-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/java.util.Collections$UnmodifiableCollection.contains(Collections.java:1068)
at java.base/java.util.stream.ReduceOps$3.getOpFlags(ReduceOps.java:185)
at java.base/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:265)
at java.base/java.util.stream.ReferencePipeline.collect(ReferencePipeline.java:702)
at org.apache.kafka.clients.ClusterConnectionStates.nodesWithConnectionSetupTimeout(ClusterConnectionStates.java:460)
at org.apache.kafka.clients.NetworkClient.handleTimedOutConnections(NetworkClient.java:890)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:609)
at org.apache.kafka.server.util.InterBrokerSendThread.pollOnce(InterBrokerSendThread.java:110)
at org.apache.kafka.server.util.InterBrokerSendThread.doWork(InterBrokerSendThread.java:137)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
13) Thread[id=76, name=data-plane-kafka-socket-acceptor-ListenerName(PLAINTEXT)-PLAINTEXT-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EventFD.set0(Native Method)
at java.base/sun.nio.ch.EventFD.set(EventFD.java:48)
at java.base/sun.nio.ch.EPollSelectorImpl.wakeup(EPollSelectorImpl.java:257)
at java.base/java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:231)
at java.base/java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:237)
at java.base/sun.nio.ch.SelectorImpl.begin(SelectorImpl.java:100)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:115)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at kafka.network.Acceptor.acceptNewConnections(SocketServer.scala:745)
at kafka.network.Acceptor.run(SocketServer.scala:693)
at java.base/java.lang.Thread.run(Thread.java:1570)
14) Thread[id=524, name=Curator-Framework-0, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at java.base/java.util.concurrent.DelayQueue.take(DelayQueue.java:242)
at java.base/java.util.concurrent.DelayQueue.take(DelayQueue.java:100)
at org.apache.curator.framework.imps.CuratorFrameworkImpl.backgroundOperationsLoop(CuratorFrameworkImpl.java:796)
at org.apache.curator.framework.imps.CuratorFrameworkImpl.access$300(CuratorFrameworkImpl.java:77)
at org.apache.curator.framework.imps.CuratorFrameworkImpl$4.call(CuratorFrameworkImpl.java:320)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
15) Thread[id=77, name=data-plane-kafka-network-thread-0-ListenerName(PLAINTEXT)-PLAINTEXT-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:112)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at kafka.network.Processor.poll(SocketServer.scala:1110)
at kafka.network.Processor.run(SocketServer.scala:1014)
at java.base/java.lang.Thread.run(Thread.java:1570)
16) Thread[id=98, name=transaction-log-manager-0, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
17) Thread[id=62, name=NIOWorkerThread-3, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
18) Thread[id=747, name=zkCallback-255-thread-2, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:410)
at java.base/java.util.concurrent.LinkedTransferQueue$DualNode.await(LinkedTransferQueue.java:452)
at java.base/java.util.concurrent.SynchronousQueue$Transferer.xferLifo(SynchronousQueue.java:194)
at java.base/java.util.concurrent.SynchronousQueue.xfer(SynchronousQueue.java:233)
at java.base/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:336)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1069)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
19) Thread[id=527, name=HttpClient@79e42938-scheduler-1, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
20) Thread[id=520, name=h2sc-253-thread-2-processing-collection1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:182)
at org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:189)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:616)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:550)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produceTask(AdaptiveExecutionStrategy.java:526)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:251)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produce(AdaptiveExecutionStrategy.java:196)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:408)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
21) Thread[id=709, name=h2sc-253-thread-18, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
22) Thread[id=538, name=h2sc-253-thread-10, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
23) Thread[id=58, name=SUITE-SolrAndKafkaReindexTest-seed#[CEE9CF5DFD9E00B9]-worker-SendThread(127.0.0.1:33335), state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/java.lang.Thread.sleepNanos0(Native Method)
at java.base/java.lang.Thread.sleepNanos(Thread.java:491)
at java.base/java.lang.Thread.sleep(Thread.java:522)
at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:362)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1197)
24) Thread[id=60, name=NIOWorkerThread-1, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
25) Thread[id=707, name=h2sc-253-thread-16, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
26) Thread[id=530, name=h2sc-253-thread-5, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
27) Thread[id=708, name=h2sc-253-thread-17, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
28) Thread[id=529, name=h2sc-253-thread-4, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
29) Thread[id=540, name=h2sc-253-thread-12, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
30) Thread[id=47, name=NIOServerCxnFactory.SelectorThread-1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/java.lang.Thread.holdsLock(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:104)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.select(NIOServerCnxnFactory.java:403)
at org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.run(NIOServerCnxnFactory.java:368)
31) Thread[id=519, name=h2sc-253-thread-1-processing-collection1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:182)
at org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:189)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:616)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:550)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produceTask(AdaptiveExecutionStrategy.java:526)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:251)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produce(AdaptiveExecutionStrategy.java:196)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:408)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
32) Thread[id=96, name=group-metadata-manager-0, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1758)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
33) Thread[id=528, name=h2sc-253-thread-3, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
34) Thread[id=75, name=zk-broker-0-to-controller-forwarding-channel-manager, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at org.apache.kafka.common.metrics.Sensor.recordInternal(Sensor.java:232)
at org.apache.kafka.common.metrics.Sensor.record(Sensor.java:226)
at org.apache.kafka.common.network.Selector.poll(Selector.java:487)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:596)
at org.apache.kafka.server.util.InterBrokerSendThread.pollOnce(InterBrokerSendThread.java:110)
at kafka.server.NodeToControllerRequestThread.doWork(NodeToControllerChannelManager.scala:377)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
35) Thread[id=627, name=executor-Fetch, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
36) Thread[id=61, name=NIOWorkerThread-2, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
37) Thread[id=517, name=executor-Rebalance, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
38) Thread[id=48, name=NIOServerCxnFactory.AcceptThread:/127.0.0.1:0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/java.lang.Thread.holdsLock(Native Method)
at java.base/sun.nio.ch.SelectorImpl.processDeregisterQueue(SelectorImpl.java:263)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:140)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.apache.zookeeper.server.NIOServerCnxnFactory$AcceptThread.select(NIOServerCnxnFactory.java:205)
at org.apache.zookeeper.server.NIOServerCnxnFactory$AcceptThread.run(NIOServerCnxnFactory.java:181)
39) Thread[id=80, name=zk-broker-0-to-controller-alter-partition-channel-manager, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/java.util.stream.AbstractPipeline.wrapSink(AbstractPipeline.java:594)
at java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:546)
at java.base/java.util.stream.ReduceOps$ReduceOp.evaluateSequential(ReduceOps.java:921)
at java.base/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:265)
at java.base/java.util.stream.ReferencePipeline.collect(ReferencePipeline.java:702)
at org.apache.kafka.clients.ClusterConnectionStates.nodesWithConnectionSetupTimeout(ClusterConnectionStates.java:460)
at org.apache.kafka.clients.NetworkClient.handleTimedOutConnections(NetworkClient.java:890)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:609)
at org.apache.kafka.server.util.InterBrokerSendThread.pollOnce(InterBrokerSendThread.java:110)
at kafka.server.NodeToControllerRequestThread.doWork(NodeToControllerChannelManager.scala:377)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
40) Thread[id=539, name=h2sc-253-thread-11, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
41) Thread[id=46, name=NIOServerCxnFactory.SelectorThread-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EventFD.set0(Native Method)
at java.base/sun.nio.ch.EventFD.set(EventFD.java:48)
at java.base/sun.nio.ch.EPollSelectorImpl.wakeup(EPollSelectorImpl.java:257)
at java.base/java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:231)
at java.base/java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:237)
at java.base/sun.nio.ch.SelectorImpl.begin(SelectorImpl.java:100)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:115)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.select(NIOServerCnxnFactory.java:403)
at org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.run(NIOServerCnxnFactory.java:368)
42) Thread[id=532, name=h2sc-253-thread-6, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
43) Thread[id=748, name=zkCallback-255-thread-3, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:410)
at java.base/java.util.concurrent.LinkedTransferQueue$DualNode.await(LinkedTransferQueue.java:452)
at java.base/java.util.concurrent.SynchronousQueue$Transferer.xferLifo(SynchronousQueue.java:194)
at java.base/java.util.concurrent.SynchronousQueue.xfer(SynchronousQueue.java:233)
at java.base/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:336)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1069)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
44) Thread[id=544, name=h2sc-253-thread-14, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
45) Thread[id=535, name=h2sc-253-thread-7, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4011)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3959)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1707)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1570)
at __randomizedtesting.SeedInfo.seed([CEE9CF5DFD9E00B9]:0)
FAILED: org.apache.solr.filestore.TestDistribFileStore.testFileStoreManagement
Error Message:
java.lang.AssertionError: expected:<200> but was:<500>
Stack Trace:
java.lang.AssertionError: expected:<200> but was:<500>
at __randomizedtesting.SeedInfo.seed([CEE9CF5DFD9E00B9:3971BAB71C04587C]:0)
at org.junit.Assert.fail(Assert.java:89)
at org.junit.Assert.failNotEquals(Assert.java:835)
at org.junit.Assert.assertEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:633)
at org.apache.solr.filestore.TestDistribFileStore$Fetcher.call(TestDistribFileStore.java:244)
at org.apache.solr.filestore.TestDistribFileStore$Fetcher.call(TestDistribFileStore.java:232)
at org.apache.solr.filestore.TestDistribFileStore.assertResponseValues(TestDistribFileStore.java:288)
at org.apache.solr.filestore.TestDistribFileStore.checkAllNodesForFile(TestDistribFileStore.java:215)
at org.apache.solr.filestore.TestDistribFileStore.testFileStoreManagement(TestDistribFileStore.java:200)
at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103)
at java.base/java.lang.reflect.Method.invoke(Method.java:580)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1763)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48)
at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
at org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.solr.util.LogLevelTestRule$LogLevelStatement.evaluate(LogLevelTestRule.java:63)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
at org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
at java.base/java.lang.Thread.run(Thread.java:1570)
[JENKINS] Solr-main-Linux (64bit/hotspot/jdk-25.0.2) - Build # 33880 - Still Unstable!
Posted by Policeman Jenkins Server via builds <bu...@solr.apache.org>.
Build: https://jenkins.thetaphi.de/job/Solr-main-Linux/33880/
Java: 64bit/hotspot/jdk-25.0.2 -XX:+UseCompressedOops -XX:+UseParallelGC
1 tests failed.
FAILED: org.apache.solr.gcs.GCSIncrementalBackupTest.testSkipConfigset
Error Message:
org.apache.solr.common.SolrException: Could not find a healthy node to handle the request.
Stack Trace:
org.apache.solr.common.SolrException: Could not find a healthy node to handle the request.
at __randomizedtesting.SeedInfo.seed([80A9DCA23FCA9EF6:8B5043ABA788C8E1]:0)
at app//org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1557)
at app//org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1199)
at app//org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1384)
at app//org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1384)
at app//org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1384)
at app//org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1375)
at app//org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1360)
at app//org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:1121)
at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:302)
at app//org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:927)
at app//org.apache.solr.cloud.api.collections.AbstractIncrementalBackupTest.testSkipConfigset(AbstractIncrementalBackupTest.java:421)
at java.base@25.0.2/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
at java.base@25.0.2/java.lang.reflect.Method.invoke(Method.java:565)
at app//com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1763)
at app//com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
at app//com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
at app//com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
at app//org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48)
at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
at app//org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
at app//org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
at app//org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at app//org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at app//org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
at app//org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
at app//org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
at app//org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
at java.base@25.0.2/java.lang.Thread.run(Thread.java:1474)