You are viewing a plain text version of this content. The canonical link for it is here.
Posted to builds@solr.apache.org by Policeman Jenkins Server via builds <bu...@solr.apache.org> on 2026/04/25 11:18:17 UTC
[JENKINS-EA] Solr-main-Windows (64bit/hotspot/jdk-27-ea+14) - Build # 8957 - Still Failing!
Build: https://jenkins.thetaphi.de/job/Solr-main-Windows/8957/
Java: 64bit/hotspot/jdk-27-ea+14 -XX:+UseCompressedOops -XX:+UseParallelGC
6 tests failed.
FAILED: org.apache.solr.crossdc.manager.SolrAndKafkaReindexTest.classMethod
Error Message:
java.lang.Exception: Error shutting down MiniSolrCloudCluster
Stack Trace:
java.lang.Exception: Error shutting down MiniSolrCloudCluster
at __randomizedtesting.SeedInfo.seed([77E68AE857775F98]:0)
at org.apache.solr.cloud.MiniSolrCloudCluster.checkForExceptions(MiniSolrCloudCluster.java:748)
at org.apache.solr.cloud.MiniSolrCloudCluster.shutdown(MiniSolrCloudCluster.java:669)
at org.apache.solr.crossdc.manager.SolrAndKafkaReindexTest.afterSolrAndKafkaIntegrationTest(SolrAndKafkaReindexTest.java:134)
at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
at java.base/java.lang.reflect.Method.invoke(Method.java:565)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1763)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:909)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
at org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
at java.base/java.lang.Thread.run(Thread.java:1527)
Suppressed: java.lang.RuntimeException: Timeout waiting for pool org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor@77ff88f4[Shutting down, pool size = 1, active threads = 1, queued tasks = 0, completed tasks = 4] to shutdown.
at org.apache.solr.common.util.ExecutorUtil.awaitTermination(ExecutorUtil.java:166)
at org.apache.solr.common.util.ExecutorUtil.awaitTermination(ExecutorUtil.java:153)
at org.apache.solr.common.util.ExecutorUtil.shutdownAndAwaitTermination(ExecutorUtil.java:119)
at org.apache.solr.core.CoreContainer.shutdown(CoreContainer.java:1325)
at org.apache.solr.servlet.CoreContainerProvider.close(CoreContainerProvider.java:132)
at org.apache.solr.servlet.CoreContainerProvider.contextDestroyed(CoreContainerProvider.java:90)
at org.eclipse.jetty.ee10.servlet.ServletContextHandler.callContextDestroyed(ServletContextHandler.java:1608)
at org.eclipse.jetty.ee10.servlet.ServletContextHandler.contextDestroyed(ServletContextHandler.java:529)
at org.eclipse.jetty.ee10.servlet.ServletHandler.doStop(ServletHandler.java:329)
at org.eclipse.jetty.util.component.AbstractLifeCycle.stop(AbstractLifeCycle.java:132)
at org.eclipse.jetty.util.component.ContainerLifeCycle.stop(ContainerLifeCycle.java:180)
at org.eclipse.jetty.util.component.ContainerLifeCycle.doStop(ContainerLifeCycle.java:202)
at org.eclipse.jetty.server.Handler$Abstract.doStop(Handler.java:499)
at org.eclipse.jetty.server.handler.ContextHandler.lambda$doStop$0(ContextHandler.java:841)
at org.eclipse.jetty.server.handler.ContextHandler$ScopedContext.call(ContextHandler.java:1488)
at org.eclipse.jetty.server.handler.ContextHandler.doStop(ContextHandler.java:841)
at org.eclipse.jetty.ee10.servlet.ServletContextHandler.stopContext(ServletContextHandler.java:1334)
at org.eclipse.jetty.ee10.servlet.ServletContextHandler.doStop(ServletContextHandler.java:1085)
at org.eclipse.jetty.util.component.AbstractLifeCycle.stop(AbstractLifeCycle.java:132)
at org.eclipse.jetty.util.component.ContainerLifeCycle.stop(ContainerLifeCycle.java:180)
at org.eclipse.jetty.util.component.ContainerLifeCycle.doStop(ContainerLifeCycle.java:202)
at org.eclipse.jetty.server.Handler$Abstract.doStop(Handler.java:499)
at org.eclipse.jetty.util.component.AbstractLifeCycle.stop(AbstractLifeCycle.java:132)
at org.eclipse.jetty.util.component.ContainerLifeCycle.stop(ContainerLifeCycle.java:180)
at org.eclipse.jetty.util.component.ContainerLifeCycle.doStop(ContainerLifeCycle.java:202)
at org.eclipse.jetty.server.Handler$Abstract.doStop(Handler.java:499)
at org.eclipse.jetty.util.component.AbstractLifeCycle.stop(AbstractLifeCycle.java:132)
at org.eclipse.jetty.util.component.ContainerLifeCycle.stop(ContainerLifeCycle.java:180)
at org.eclipse.jetty.util.component.ContainerLifeCycle.doStop(ContainerLifeCycle.java:202)
at org.eclipse.jetty.server.Handler$Abstract.doStop(Handler.java:499)
at org.eclipse.jetty.server.Server.doStop(Server.java:690)
at org.eclipse.jetty.util.component.AbstractLifeCycle.stop(AbstractLifeCycle.java:132)
at org.apache.solr.embedded.JettySolrRunner.stop(JettySolrRunner.java:586)
at org.apache.solr.cloud.MiniSolrCloudCluster.stopJettySolrRunner(MiniSolrCloudCluster.java:561)
at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$shutdown$1(MiniSolrCloudCluster.java:661)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:330)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:381)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1090)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
... 1 more
FAILED: org.apache.solr.crossdc.manager.SolrAndKafkaReindexTest.classMethod
Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 67 threads leaked from SUITE scope at org.apache.solr.crossdc.manager.SolrAndKafkaReindexTest:
1) Thread[id=84, name=ExpirationReaper-0-Heartbeat, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:301)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
2) Thread[id=40, name=SyncThread:0, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at org.apache.zookeeper.server.SyncRequestProcessor.run(SyncRequestProcessor.java:170)
3) Thread[id=96, name=data-plane-kafka-request-handler-6, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:436)
at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:484)
at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:113)
at java.base/java.lang.Thread.run(Thread.java:1527)
4) Thread[id=71, name=AddPartitionsToTxnSenderThread-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:596)
at org.apache.kafka.server.util.InterBrokerSendThread.pollOnce(InterBrokerSendThread.java:110)
at org.apache.kafka.server.util.InterBrokerSendThread.doWork(InterBrokerSendThread.java:137)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
5) Thread[id=89, name=ExpirationReaper-0-AlterAcls, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:292)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
6) Thread[id=80, name=LogDirFailureHandler, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:421)
at org.apache.kafka.storage.internals.log.LogDirFailureChannel.takeNextOfflineLogDir(LogDirFailureChannel.java:75)
at kafka.server.ReplicaManager$LogDirFailureHandler.doWork(ReplicaManager.scala:320)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
7) Thread[id=67, name=data-plane-kafka-network-thread-0-ListenerName(PLAINTEXT)-PLAINTEXT-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at kafka.network.Processor.poll(SocketServer.scala:1110)
at kafka.network.Processor.run(SocketServer.scala:1014)
at java.base/java.lang.Thread.run(Thread.java:1527)
8) Thread[id=91, name=data-plane-kafka-request-handler-1, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:436)
at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:484)
at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:113)
at java.base/java.lang.Thread.run(Thread.java:1527)
9) Thread[id=82, name=controller-event-thread, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
at kafka.controller.ControllerEventManager.kafka$controller$ControllerEventManager$$pollFromEventQueue(ControllerEventManager.scala:156)
at kafka.controller.ControllerEventManager$ControllerEventThread.doWork(ControllerEventManager.scala:129)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
10) Thread[id=54, name=ThrottledChannelReaper-Fetch, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:292)
at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:178)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
11) Thread[id=86, name=group-metadata-manager-0, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1166)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:883)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
12) Thread[id=88, name=transaction-log-manager-0, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1166)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:883)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
13) Thread[id=36, name=NIOServerCxnFactory.SelectorThread-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.select(NIOServerCnxnFactory.java:403)
at org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.run(NIOServerCnxnFactory.java:368)
14) Thread[id=75, name=ExpirationReaper-0-ElectLeader, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:292)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
15) Thread[id=97, name=data-plane-kafka-request-handler-7, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:436)
at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:484)
at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:113)
at java.base/java.lang.Thread.run(Thread.java:1527)
16) Thread[id=87, name=TxnMarkerSenderThread-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:596)
at org.apache.kafka.server.util.InterBrokerSendThread.pollOnce(InterBrokerSendThread.java:110)
at org.apache.kafka.server.util.InterBrokerSendThread.doWork(InterBrokerSendThread.java:137)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
17) Thread[id=90, name=data-plane-kafka-request-handler-0, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:436)
at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:484)
at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:113)
at java.base/java.lang.Thread.run(Thread.java:1527)
18) Thread[id=505, name=h2sc-249-thread-8, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
19) Thread[id=99, name=/config/changes-event-process-thread, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at kafka.common.ZkNodeChangeNotificationListener$ChangeEventProcessThread.doWork(ZkNodeChangeNotificationListener.scala:147)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
20) Thread[id=95, name=data-plane-kafka-request-handler-5, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:436)
at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:484)
at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:113)
at java.base/java.lang.Thread.run(Thread.java:1527)
21) Thread[id=93, name=data-plane-kafka-request-handler-3, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:436)
at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:484)
at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:113)
at java.base/java.lang.Thread.run(Thread.java:1527)
22) Thread[id=73, name=ExpirationReaper-0-Fetch, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:301)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
23) Thread[id=41, name=ProcessThread(sid:0 cport:49533):, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:142)
24) Thread[id=69, name=data-plane-kafka-network-thread-0-ListenerName(PLAINTEXT)-PLAINTEXT-2, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at kafka.network.Processor.poll(SocketServer.scala:1110)
at kafka.network.Processor.run(SocketServer.scala:1014)
at java.base/java.lang.Thread.run(Thread.java:1527)
25) Thread[id=66, name=data-plane-kafka-socket-acceptor-ListenerName(PLAINTEXT)-PLAINTEXT-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at kafka.network.Acceptor.acceptNewConnections(SocketServer.scala:745)
at kafka.network.Acceptor.run(SocketServer.scala:693)
at java.base/java.lang.Thread.run(Thread.java:1527)
26) Thread[id=466, name=executor-Rebalance, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
27) Thread[id=455, name=CrossDCConsumer-242-thread-1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:596)
at org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.poll(ConsumerNetworkClient.java:282)
at org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.poll(ConsumerNetworkClient.java:253)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:692)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.solr.crossdc.manager.consumer.KafkaCrossDcConsumer.pollAndProcessRequests(KafkaCrossDcConsumer.java:366)
at org.apache.solr.crossdc.manager.consumer.KafkaCrossDcConsumer.run(KafkaCrossDcConsumer.java:285)
at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:545)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:330)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:381)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1090)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
28) Thread[id=83, name=ExpirationReaper-0-topic, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:292)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
29) Thread[id=687, name=zkCallback-251-thread-3, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:408)
at java.base/java.util.concurrent.LinkedTransferQueue$DualNode.await(LinkedTransferQueue.java:452)
at java.base/java.util.concurrent.SynchronousQueue$Transferer.xferLifo(SynchronousQueue.java:194)
at java.base/java.util.concurrent.SynchronousQueue.xfer(SynchronousQueue.java:235)
at java.base/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:338)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1015)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
30) Thread[id=484, name=h2sc-249-thread-2-processing-collection1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:182)
at org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:189)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:616)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:550)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produceTask(AdaptiveExecutionStrategy.java:526)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:251)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produce(AdaptiveExecutionStrategy.java:196)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:381)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1090)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
31) Thread[id=486, name=Curator-ConnectionStateManager-0, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:436)
at org.apache.curator.framework.state.ConnectionStateManager.processEvents(ConnectionStateManager.java:246)
at org.apache.curator.framework.state.ConnectionStateManager.lambda$start$0(ConnectionStateManager.java:127)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:330)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1090)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
32) Thread[id=48, name=SUITE-SolrAndKafkaReindexTest-seed#[77E68AE857775F98]-worker-SendThread(127.0.0.1:49533), state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1290)
33) Thread[id=65, name=zk-broker-0-to-controller-forwarding-channel-manager, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:596)
at org.apache.kafka.server.util.InterBrokerSendThread.pollOnce(InterBrokerSendThread.java:110)
at kafka.server.NodeToControllerRequestThread.doWork(NodeToControllerChannelManager.scala:377)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
34) Thread[id=527, name=executor-Fetch, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
35) Thread[id=504, name=h2sc-249-thread-7, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
36) Thread[id=74, name=ExpirationReaper-0-DeleteRecords, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:292)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
37) Thread[id=51, name=NIOWorkerThread-2, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
38) Thread[id=92, name=data-plane-kafka-request-handler-2, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:436)
at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:484)
at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:113)
at java.base/java.lang.Thread.run(Thread.java:1527)
39) Thread[id=499, name=h2sc-249-thread-3, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
40) Thread[id=614, name=h2sc-249-thread-10, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
41) Thread[id=628, name=zkCallback-251-thread-1, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:408)
at java.base/java.util.concurrent.LinkedTransferQueue$DualNode.await(LinkedTransferQueue.java:452)
at java.base/java.util.concurrent.SynchronousQueue$Transferer.xferLifo(SynchronousQueue.java:194)
at java.base/java.util.concurrent.SynchronousQueue.xfer(SynchronousQueue.java:235)
at java.base/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:338)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1015)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
42) Thread[id=52, name=NIOWorkerThread-3, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
43) Thread[id=42, name=RequestThrottler, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at org.apache.zookeeper.server.RequestThrottler.run(RequestThrottler.java:147)
44) Thread[id=55, name=ThrottledChannelReaper-Produce, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:292)
at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:178)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
45) Thread[id=64, name=feature-zk-node-event-process-thread, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at kafka.server.FinalizedFeatureChangeListener$ChangeNotificationProcessorThread.doWork(FinalizedFeatureChangeListener.scala:154)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
46) Thread[id=37, name=NIOServerCxnFactory.SelectorThread-1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.select(NIOServerCnxnFactory.java:403)
at org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.run(NIOServerCnxnFactory.java:368)
47) Thread[id=70, name=zk-broker-0-to-controller-alter-partition-channel-manager, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:596)
at org.apache.kafka.server.util.InterBrokerSendThread.pollOnce(InterBrokerSendThread.java:110)
at kafka.server.NodeToControllerRequestThread.doWork(NodeToControllerChannelManager.scala:377)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
48) Thread[id=98, name=Controller-0-to-broker-0-send-thread, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:242)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
49) Thread[id=68, name=data-plane-kafka-network-thread-0-ListenerName(PLAINTEXT)-PLAINTEXT-1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at kafka.network.Processor.poll(SocketServer.scala:1110)
at kafka.network.Processor.run(SocketServer.scala:1014)
at java.base/java.lang.Thread.run(Thread.java:1527)
50) Thread[id=501, name=h2sc-249-thread-4, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
51) Thread[id=72, name=ExpirationReaper-0-Produce, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:292)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
52) Thread[id=57, name=ThrottledChannelReaper-ControllerMutation, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:292)
at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:178)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
53) Thread[id=686, name=zkCallback-251-thread-2, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:408)
at java.base/java.util.concurrent.LinkedTransferQueue$DualNode.await(LinkedTransferQueue.java:452)
at java.base/java.util.concurrent.SynchronousQueue$Transferer.xferLifo(SynchronousQueue.java:194)
at java.base/java.util.concurrent.SynchronousQueue.xfer(SynchronousQueue.java:235)
at java.base/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:338)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1015)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
54) Thread[id=490, name=Curator-Framework-0, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.DelayQueue.take(DelayQueue.java:243)
at java.base/java.util.concurrent.DelayQueue.take(DelayQueue.java:100)
at org.apache.curator.framework.imps.CuratorFrameworkImpl.backgroundOperationsLoop(CuratorFrameworkImpl.java:796)
at org.apache.curator.framework.imps.CuratorFrameworkImpl.access$300(CuratorFrameworkImpl.java:77)
at org.apache.curator.framework.imps.CuratorFrameworkImpl$4.call(CuratorFrameworkImpl.java:320)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:330)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:309)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1090)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
55) Thread[id=76, name=ExpirationReaper-0-RemoteFetch, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:292)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
56) Thread[id=502, name=h2sc-249-thread-5, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
57) Thread[id=56, name=ThrottledChannelReaper-Request, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:292)
at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:178)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
58) Thread[id=498, name=HttpClient@5840ded8-scheduler-1, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1166)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:883)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
59) Thread[id=94, name=data-plane-kafka-request-handler-4, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:436)
at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:484)
at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:113)
at java.base/java.lang.Thread.run(Thread.java:1527)
60) Thread[id=609, name=h2sc-249-thread-9, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
61) Thread[id=50, name=NIOWorkerThread-1, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
62) Thread[id=49, name=SUITE-SolrAndKafkaReindexTest-seed#[77E68AE857775F98]-worker-EventThread, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:537)
63) Thread[id=482, name=h2sc-249-thread-1-processing-collection1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:182)
at org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:189)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:616)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:550)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produceTask(AdaptiveExecutionStrategy.java:526)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:251)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produce(AdaptiveExecutionStrategy.java:196)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:381)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1090)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
64) Thread[id=38, name=NIOServerCxnFactory.AcceptThread:/127.0.0.1:0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.apache.zookeeper.server.NIOServerCnxnFactory$AcceptThread.select(NIOServerCnxnFactory.java:205)
at org.apache.zookeeper.server.NIOServerCnxnFactory$AcceptThread.run(NIOServerCnxnFactory.java:181)
65) Thread[id=53, name=SensorExpiryThread, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1166)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:883)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
66) Thread[id=503, name=h2sc-249-thread-6, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
67) Thread[id=85, name=ExpirationReaper-0-Rebalance, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:292)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 67 threads leaked from SUITE scope at org.apache.solr.crossdc.manager.SolrAndKafkaReindexTest:
1) Thread[id=84, name=ExpirationReaper-0-Heartbeat, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:301)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
2) Thread[id=40, name=SyncThread:0, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at org.apache.zookeeper.server.SyncRequestProcessor.run(SyncRequestProcessor.java:170)
3) Thread[id=96, name=data-plane-kafka-request-handler-6, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:436)
at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:484)
at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:113)
at java.base/java.lang.Thread.run(Thread.java:1527)
4) Thread[id=71, name=AddPartitionsToTxnSenderThread-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:596)
at org.apache.kafka.server.util.InterBrokerSendThread.pollOnce(InterBrokerSendThread.java:110)
at org.apache.kafka.server.util.InterBrokerSendThread.doWork(InterBrokerSendThread.java:137)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
5) Thread[id=89, name=ExpirationReaper-0-AlterAcls, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:292)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
6) Thread[id=80, name=LogDirFailureHandler, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:421)
at org.apache.kafka.storage.internals.log.LogDirFailureChannel.takeNextOfflineLogDir(LogDirFailureChannel.java:75)
at kafka.server.ReplicaManager$LogDirFailureHandler.doWork(ReplicaManager.scala:320)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
7) Thread[id=67, name=data-plane-kafka-network-thread-0-ListenerName(PLAINTEXT)-PLAINTEXT-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at kafka.network.Processor.poll(SocketServer.scala:1110)
at kafka.network.Processor.run(SocketServer.scala:1014)
at java.base/java.lang.Thread.run(Thread.java:1527)
8) Thread[id=91, name=data-plane-kafka-request-handler-1, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:436)
at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:484)
at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:113)
at java.base/java.lang.Thread.run(Thread.java:1527)
9) Thread[id=82, name=controller-event-thread, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
at kafka.controller.ControllerEventManager.kafka$controller$ControllerEventManager$$pollFromEventQueue(ControllerEventManager.scala:156)
at kafka.controller.ControllerEventManager$ControllerEventThread.doWork(ControllerEventManager.scala:129)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
10) Thread[id=54, name=ThrottledChannelReaper-Fetch, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:292)
at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:178)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
11) Thread[id=86, name=group-metadata-manager-0, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1166)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:883)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
12) Thread[id=88, name=transaction-log-manager-0, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1166)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:883)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
13) Thread[id=36, name=NIOServerCxnFactory.SelectorThread-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.select(NIOServerCnxnFactory.java:403)
at org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.run(NIOServerCnxnFactory.java:368)
14) Thread[id=75, name=ExpirationReaper-0-ElectLeader, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:292)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
15) Thread[id=97, name=data-plane-kafka-request-handler-7, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:436)
at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:484)
at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:113)
at java.base/java.lang.Thread.run(Thread.java:1527)
16) Thread[id=87, name=TxnMarkerSenderThread-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:596)
at org.apache.kafka.server.util.InterBrokerSendThread.pollOnce(InterBrokerSendThread.java:110)
at org.apache.kafka.server.util.InterBrokerSendThread.doWork(InterBrokerSendThread.java:137)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
17) Thread[id=90, name=data-plane-kafka-request-handler-0, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:436)
at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:484)
at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:113)
at java.base/java.lang.Thread.run(Thread.java:1527)
18) Thread[id=505, name=h2sc-249-thread-8, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
19) Thread[id=99, name=/config/changes-event-process-thread, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at kafka.common.ZkNodeChangeNotificationListener$ChangeEventProcessThread.doWork(ZkNodeChangeNotificationListener.scala:147)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
20) Thread[id=95, name=data-plane-kafka-request-handler-5, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:436)
at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:484)
at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:113)
at java.base/java.lang.Thread.run(Thread.java:1527)
21) Thread[id=93, name=data-plane-kafka-request-handler-3, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:436)
at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:484)
at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:113)
at java.base/java.lang.Thread.run(Thread.java:1527)
22) Thread[id=73, name=ExpirationReaper-0-Fetch, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:301)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
23) Thread[id=41, name=ProcessThread(sid:0 cport:49533):, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:142)
24) Thread[id=69, name=data-plane-kafka-network-thread-0-ListenerName(PLAINTEXT)-PLAINTEXT-2, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at kafka.network.Processor.poll(SocketServer.scala:1110)
at kafka.network.Processor.run(SocketServer.scala:1014)
at java.base/java.lang.Thread.run(Thread.java:1527)
25) Thread[id=66, name=data-plane-kafka-socket-acceptor-ListenerName(PLAINTEXT)-PLAINTEXT-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at kafka.network.Acceptor.acceptNewConnections(SocketServer.scala:745)
at kafka.network.Acceptor.run(SocketServer.scala:693)
at java.base/java.lang.Thread.run(Thread.java:1527)
26) Thread[id=466, name=executor-Rebalance, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
27) Thread[id=455, name=CrossDCConsumer-242-thread-1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:596)
at org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.poll(ConsumerNetworkClient.java:282)
at org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.poll(ConsumerNetworkClient.java:253)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:692)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.solr.crossdc.manager.consumer.KafkaCrossDcConsumer.pollAndProcessRequests(KafkaCrossDcConsumer.java:366)
at org.apache.solr.crossdc.manager.consumer.KafkaCrossDcConsumer.run(KafkaCrossDcConsumer.java:285)
at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:545)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:330)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:381)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1090)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
28) Thread[id=83, name=ExpirationReaper-0-topic, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:292)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
29) Thread[id=687, name=zkCallback-251-thread-3, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:408)
at java.base/java.util.concurrent.LinkedTransferQueue$DualNode.await(LinkedTransferQueue.java:452)
at java.base/java.util.concurrent.SynchronousQueue$Transferer.xferLifo(SynchronousQueue.java:194)
at java.base/java.util.concurrent.SynchronousQueue.xfer(SynchronousQueue.java:235)
at java.base/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:338)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1015)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
30) Thread[id=484, name=h2sc-249-thread-2-processing-collection1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:182)
at org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:189)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:616)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:550)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produceTask(AdaptiveExecutionStrategy.java:526)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:251)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produce(AdaptiveExecutionStrategy.java:196)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:381)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1090)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
31) Thread[id=486, name=Curator-ConnectionStateManager-0, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:436)
at org.apache.curator.framework.state.ConnectionStateManager.processEvents(ConnectionStateManager.java:246)
at org.apache.curator.framework.state.ConnectionStateManager.lambda$start$0(ConnectionStateManager.java:127)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:330)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1090)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
32) Thread[id=48, name=SUITE-SolrAndKafkaReindexTest-seed#[77E68AE857775F98]-worker-SendThread(127.0.0.1:49533), state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1290)
33) Thread[id=65, name=zk-broker-0-to-controller-forwarding-channel-manager, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:596)
at org.apache.kafka.server.util.InterBrokerSendThread.pollOnce(InterBrokerSendThread.java:110)
at kafka.server.NodeToControllerRequestThread.doWork(NodeToControllerChannelManager.scala:377)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
34) Thread[id=527, name=executor-Fetch, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
35) Thread[id=504, name=h2sc-249-thread-7, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
36) Thread[id=74, name=ExpirationReaper-0-DeleteRecords, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:292)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
37) Thread[id=51, name=NIOWorkerThread-2, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
38) Thread[id=92, name=data-plane-kafka-request-handler-2, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:436)
at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:484)
at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:113)
at java.base/java.lang.Thread.run(Thread.java:1527)
39) Thread[id=499, name=h2sc-249-thread-3, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
40) Thread[id=614, name=h2sc-249-thread-10, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
41) Thread[id=628, name=zkCallback-251-thread-1, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:408)
at java.base/java.util.concurrent.LinkedTransferQueue$DualNode.await(LinkedTransferQueue.java:452)
at java.base/java.util.concurrent.SynchronousQueue$Transferer.xferLifo(SynchronousQueue.java:194)
at java.base/java.util.concurrent.SynchronousQueue.xfer(SynchronousQueue.java:235)
at java.base/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:338)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1015)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
42) Thread[id=52, name=NIOWorkerThread-3, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
43) Thread[id=42, name=RequestThrottler, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at org.apache.zookeeper.server.RequestThrottler.run(RequestThrottler.java:147)
44) Thread[id=55, name=ThrottledChannelReaper-Produce, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:292)
at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:178)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
45) Thread[id=64, name=feature-zk-node-event-process-thread, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at kafka.server.FinalizedFeatureChangeListener$ChangeNotificationProcessorThread.doWork(FinalizedFeatureChangeListener.scala:154)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
46) Thread[id=37, name=NIOServerCxnFactory.SelectorThread-1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.select(NIOServerCnxnFactory.java:403)
at org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.run(NIOServerCnxnFactory.java:368)
47) Thread[id=70, name=zk-broker-0-to-controller-alter-partition-channel-manager, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:596)
at org.apache.kafka.server.util.InterBrokerSendThread.pollOnce(InterBrokerSendThread.java:110)
at kafka.server.NodeToControllerRequestThread.doWork(NodeToControllerChannelManager.scala:377)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
48) Thread[id=98, name=Controller-0-to-broker-0-send-thread, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:242)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
49) Thread[id=68, name=data-plane-kafka-network-thread-0-ListenerName(PLAINTEXT)-PLAINTEXT-1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at kafka.network.Processor.poll(SocketServer.scala:1110)
at kafka.network.Processor.run(SocketServer.scala:1014)
at java.base/java.lang.Thread.run(Thread.java:1527)
50) Thread[id=501, name=h2sc-249-thread-4, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
51) Thread[id=72, name=ExpirationReaper-0-Produce, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:292)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
52) Thread[id=57, name=ThrottledChannelReaper-ControllerMutation, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:292)
at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:178)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
53) Thread[id=686, name=zkCallback-251-thread-2, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:408)
at java.base/java.util.concurrent.LinkedTransferQueue$DualNode.await(LinkedTransferQueue.java:452)
at java.base/java.util.concurrent.SynchronousQueue$Transferer.xferLifo(SynchronousQueue.java:194)
at java.base/java.util.concurrent.SynchronousQueue.xfer(SynchronousQueue.java:235)
at java.base/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:338)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1015)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
54) Thread[id=490, name=Curator-Framework-0, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.DelayQueue.take(DelayQueue.java:243)
at java.base/java.util.concurrent.DelayQueue.take(DelayQueue.java:100)
at org.apache.curator.framework.imps.CuratorFrameworkImpl.backgroundOperationsLoop(CuratorFrameworkImpl.java:796)
at org.apache.curator.framework.imps.CuratorFrameworkImpl.access$300(CuratorFrameworkImpl.java:77)
at org.apache.curator.framework.imps.CuratorFrameworkImpl$4.call(CuratorFrameworkImpl.java:320)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:330)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:309)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1090)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
55) Thread[id=76, name=ExpirationReaper-0-RemoteFetch, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:292)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
56) Thread[id=502, name=h2sc-249-thread-5, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
57) Thread[id=56, name=ThrottledChannelReaper-Request, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:292)
at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:178)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
58) Thread[id=498, name=HttpClient@5840ded8-scheduler-1, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1166)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:883)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
59) Thread[id=94, name=data-plane-kafka-request-handler-4, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:436)
at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:484)
at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:113)
at java.base/java.lang.Thread.run(Thread.java:1527)
60) Thread[id=609, name=h2sc-249-thread-9, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
61) Thread[id=50, name=NIOWorkerThread-1, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
62) Thread[id=49, name=SUITE-SolrAndKafkaReindexTest-seed#[77E68AE857775F98]-worker-EventThread, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:537)
63) Thread[id=482, name=h2sc-249-thread-1-processing-collection1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:182)
at org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:189)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:616)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:550)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produceTask(AdaptiveExecutionStrategy.java:526)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:251)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produce(AdaptiveExecutionStrategy.java:196)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:381)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1090)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
64) Thread[id=38, name=NIOServerCxnFactory.AcceptThread:/127.0.0.1:0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.apache.zookeeper.server.NIOServerCnxnFactory$AcceptThread.select(NIOServerCnxnFactory.java:205)
at org.apache.zookeeper.server.NIOServerCnxnFactory$AcceptThread.run(NIOServerCnxnFactory.java:181)
65) Thread[id=53, name=SensorExpiryThread, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1166)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:883)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
66) Thread[id=503, name=h2sc-249-thread-6, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
67) Thread[id=85, name=ExpirationReaper-0-Rebalance, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:292)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
at __randomizedtesting.SeedInfo.seed([77E68AE857775F98]:0)
FAILED: org.apache.solr.crossdc.manager.SolrAndKafkaReindexTest.classMethod
Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=65, name=zk-broker-0-to-controller-forwarding-channel-manager, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:596)
at org.apache.kafka.server.util.InterBrokerSendThread.pollOnce(InterBrokerSendThread.java:110)
at kafka.server.NodeToControllerRequestThread.doWork(NodeToControllerChannelManager.scala:377)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
2) Thread[id=527, name=executor-Fetch, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
3) Thread[id=504, name=h2sc-249-thread-7, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
4) Thread[id=71, name=AddPartitionsToTxnSenderThread-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.IOUtil.write1(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.wakeup(WEPollSelectorImpl.java:235)
at java.base/java.nio.channels.spi.AbstractSelector$1.postInterrupt(AbstractSelector.java:111)
at java.base/java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:237)
at java.base/sun.nio.ch.SelectorImpl.begin(SelectorImpl.java:100)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:105)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:596)
at org.apache.kafka.server.util.InterBrokerSendThread.pollOnce(InterBrokerSendThread.java:110)
at org.apache.kafka.server.util.InterBrokerSendThread.doWork(InterBrokerSendThread.java:137)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
5) Thread[id=67, name=data-plane-kafka-network-thread-0-ListenerName(PLAINTEXT)-PLAINTEXT-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at kafka.network.Processor.poll(SocketServer.scala:1110)
at kafka.network.Processor.run(SocketServer.scala:1014)
at java.base/java.lang.Thread.run(Thread.java:1527)
6) Thread[id=51, name=NIOWorkerThread-2, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
7) Thread[id=499, name=h2sc-249-thread-3, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
8) Thread[id=614, name=h2sc-249-thread-10, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
9) Thread[id=628, name=zkCallback-251-thread-1, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:408)
at java.base/java.util.concurrent.LinkedTransferQueue$DualNode.await(LinkedTransferQueue.java:452)
at java.base/java.util.concurrent.SynchronousQueue$Transferer.xferLifo(SynchronousQueue.java:194)
at java.base/java.util.concurrent.SynchronousQueue.xfer(SynchronousQueue.java:235)
at java.base/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:338)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1015)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
10) Thread[id=52, name=NIOWorkerThread-3, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
11) Thread[id=86, name=group-metadata-manager-0, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1166)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:883)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
12) Thread[id=88, name=transaction-log-manager-0, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1166)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:883)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
13) Thread[id=37, name=NIOServerCxnFactory.SelectorThread-1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.select(NIOServerCnxnFactory.java:403)
at org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.run(NIOServerCnxnFactory.java:368)
14) Thread[id=36, name=NIOServerCxnFactory.SelectorThread-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.select(NIOServerCnxnFactory.java:403)
at org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.run(NIOServerCnxnFactory.java:368)
15) Thread[id=87, name=TxnMarkerSenderThread-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.IOUtil.write1(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.wakeup(WEPollSelectorImpl.java:235)
at java.base/java.nio.channels.spi.AbstractSelector$1.postInterrupt(AbstractSelector.java:111)
at java.base/java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:237)
at java.base/sun.nio.ch.SelectorImpl.begin(SelectorImpl.java:100)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:105)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:596)
at org.apache.kafka.server.util.InterBrokerSendThread.pollOnce(InterBrokerSendThread.java:110)
at org.apache.kafka.server.util.InterBrokerSendThread.doWork(InterBrokerSendThread.java:137)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
16) Thread[id=70, name=zk-broker-0-to-controller-alter-partition-channel-manager, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:596)
at org.apache.kafka.server.util.InterBrokerSendThread.pollOnce(InterBrokerSendThread.java:110)
at kafka.server.NodeToControllerRequestThread.doWork(NodeToControllerChannelManager.scala:377)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
17) Thread[id=505, name=h2sc-249-thread-8, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
18) Thread[id=68, name=data-plane-kafka-network-thread-0-ListenerName(PLAINTEXT)-PLAINTEXT-1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at kafka.network.Processor.poll(SocketServer.scala:1110)
at kafka.network.Processor.run(SocketServer.scala:1014)
at java.base/java.lang.Thread.run(Thread.java:1527)
19) Thread[id=501, name=h2sc-249-thread-4, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
20) Thread[id=686, name=zkCallback-251-thread-2, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:408)
at java.base/java.util.concurrent.LinkedTransferQueue$DualNode.await(LinkedTransferQueue.java:452)
at java.base/java.util.concurrent.SynchronousQueue$Transferer.xferLifo(SynchronousQueue.java:194)
at java.base/java.util.concurrent.SynchronousQueue.xfer(SynchronousQueue.java:235)
at java.base/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:338)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1015)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
21) Thread[id=490, name=Curator-Framework-0, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.DelayQueue.take(DelayQueue.java:243)
at java.base/java.util.concurrent.DelayQueue.take(DelayQueue.java:100)
at org.apache.curator.framework.imps.CuratorFrameworkImpl.backgroundOperationsLoop(CuratorFrameworkImpl.java:796)
at org.apache.curator.framework.imps.CuratorFrameworkImpl.access$300(CuratorFrameworkImpl.java:77)
at org.apache.curator.framework.imps.CuratorFrameworkImpl$4.call(CuratorFrameworkImpl.java:320)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:330)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:309)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1090)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
22) Thread[id=502, name=h2sc-249-thread-5, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
23) Thread[id=498, name=HttpClient@5840ded8-scheduler-1, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1166)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:883)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
24) Thread[id=609, name=h2sc-249-thread-9, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
25) Thread[id=50, name=NIOWorkerThread-1, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
26) Thread[id=482, name=h2sc-249-thread-1-processing-collection1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:182)
at org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:189)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:616)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:550)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produceTask(AdaptiveExecutionStrategy.java:526)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:251)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produce(AdaptiveExecutionStrategy.java:196)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:381)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1090)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
27) Thread[id=69, name=data-plane-kafka-network-thread-0-ListenerName(PLAINTEXT)-PLAINTEXT-2, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.IOUtil.write1(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.wakeup(WEPollSelectorImpl.java:235)
at java.base/java.nio.channels.spi.AbstractSelector$1.postInterrupt(AbstractSelector.java:111)
at java.base/java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:237)
at java.base/sun.nio.ch.SelectorImpl.begin(SelectorImpl.java:100)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:105)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at kafka.network.Processor.poll(SocketServer.scala:1110)
at kafka.network.Processor.run(SocketServer.scala:1014)
at java.base/java.lang.Thread.run(Thread.java:1527)
28) Thread[id=66, name=data-plane-kafka-socket-acceptor-ListenerName(PLAINTEXT)-PLAINTEXT-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at kafka.network.Acceptor.run(SocketServer.scala:694)
at java.base/java.lang.Thread.run(Thread.java:1527)
29) Thread[id=466, name=executor-Rebalance, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
30) Thread[id=455, name=CrossDCConsumer-242-thread-1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.poll(ConsumerNetworkClient.java:269)
at org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.poll(ConsumerNetworkClient.java:232)
at org.apache.kafka.clients.consumer.internals.AbstractCoordinator.ensureCoordinatorReady(AbstractCoordinator.java:290)
at org.apache.kafka.clients.consumer.internals.AbstractCoordinator.ensureCoordinatorReady(AbstractCoordinator.java:264)
at org.apache.kafka.clients.consumer.internals.ConsumerCoordinator.coordinatorUnknownAndUnreadySync(ConsumerCoordinator.java:451)
at org.apache.kafka.clients.consumer.internals.ConsumerCoordinator.poll(ConsumerCoordinator.java:483)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.updateAssignmentMetadataIfNeeded(ClassicKafkaConsumer.java:657)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:616)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.solr.crossdc.manager.consumer.KafkaCrossDcConsumer.pollAndProcessRequests(KafkaCrossDcConsumer.java:366)
at org.apache.solr.crossdc.manager.consumer.KafkaCrossDcConsumer.run(KafkaCrossDcConsumer.java:285)
at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:545)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:330)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:381)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1090)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
31) Thread[id=687, name=zkCallback-251-thread-3, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:408)
at java.base/java.util.concurrent.LinkedTransferQueue$DualNode.await(LinkedTransferQueue.java:452)
at java.base/java.util.concurrent.SynchronousQueue$Transferer.xferLifo(SynchronousQueue.java:194)
at java.base/java.util.concurrent.SynchronousQueue.xfer(SynchronousQueue.java:235)
at java.base/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:338)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1015)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
32) Thread[id=38, name=NIOServerCxnFactory.AcceptThread:/127.0.0.1:0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.IOUtil.drain(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.clearInterrupt(WEPollSelectorImpl.java:247)
at java.base/sun.nio.ch.WEPollSelectorImpl.processEvents(WEPollSelectorImpl.java:186)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:116)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.apache.zookeeper.server.NIOServerCnxnFactory$AcceptThread.select(NIOServerCnxnFactory.java:205)
at org.apache.zookeeper.server.NIOServerCnxnFactory$AcceptThread.run(NIOServerCnxnFactory.java:181)
33) Thread[id=484, name=h2sc-249-thread-2-processing-collection1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:182)
at org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:189)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:616)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:550)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produceTask(AdaptiveExecutionStrategy.java:526)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:251)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produce(AdaptiveExecutionStrategy.java:196)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:381)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1090)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
34) Thread[id=486, name=Curator-ConnectionStateManager-0, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:436)
at org.apache.curator.framework.state.ConnectionStateManager.processEvents(ConnectionStateManager.java:246)
at org.apache.curator.framework.state.ConnectionStateManager.lambda$start$0(ConnectionStateManager.java:127)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:330)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1090)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
35) Thread[id=53, name=SensorExpiryThread, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1166)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:883)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
36) Thread[id=48, name=SUITE-SolrAndKafkaReindexTest-seed#[77E68AE857775F98]-worker-SendThread(127.0.0.1:49533), state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/java.lang.Thread.sleepNanos0(Native Method)
at java.base/java.lang.Thread.sleepNanos(Thread.java:562)
at java.base/java.lang.Thread.sleep(Thread.java:593)
at org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:214)
at org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1374)
at org.apache.zookeeper.ClientCnxn$SendThread.cleanAndNotifyState(ClientCnxn.java:1342)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1310)
37) Thread[id=503, name=h2sc-249-thread-6, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=65, name=zk-broker-0-to-controller-forwarding-channel-manager, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:596)
at org.apache.kafka.server.util.InterBrokerSendThread.pollOnce(InterBrokerSendThread.java:110)
at kafka.server.NodeToControllerRequestThread.doWork(NodeToControllerChannelManager.scala:377)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
2) Thread[id=527, name=executor-Fetch, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
3) Thread[id=504, name=h2sc-249-thread-7, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
4) Thread[id=71, name=AddPartitionsToTxnSenderThread-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.IOUtil.write1(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.wakeup(WEPollSelectorImpl.java:235)
at java.base/java.nio.channels.spi.AbstractSelector$1.postInterrupt(AbstractSelector.java:111)
at java.base/java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:237)
at java.base/sun.nio.ch.SelectorImpl.begin(SelectorImpl.java:100)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:105)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:596)
at org.apache.kafka.server.util.InterBrokerSendThread.pollOnce(InterBrokerSendThread.java:110)
at org.apache.kafka.server.util.InterBrokerSendThread.doWork(InterBrokerSendThread.java:137)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
5) Thread[id=67, name=data-plane-kafka-network-thread-0-ListenerName(PLAINTEXT)-PLAINTEXT-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at kafka.network.Processor.poll(SocketServer.scala:1110)
at kafka.network.Processor.run(SocketServer.scala:1014)
at java.base/java.lang.Thread.run(Thread.java:1527)
6) Thread[id=51, name=NIOWorkerThread-2, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
7) Thread[id=499, name=h2sc-249-thread-3, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
8) Thread[id=614, name=h2sc-249-thread-10, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
9) Thread[id=628, name=zkCallback-251-thread-1, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:408)
at java.base/java.util.concurrent.LinkedTransferQueue$DualNode.await(LinkedTransferQueue.java:452)
at java.base/java.util.concurrent.SynchronousQueue$Transferer.xferLifo(SynchronousQueue.java:194)
at java.base/java.util.concurrent.SynchronousQueue.xfer(SynchronousQueue.java:235)
at java.base/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:338)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1015)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
10) Thread[id=52, name=NIOWorkerThread-3, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
11) Thread[id=86, name=group-metadata-manager-0, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1166)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:883)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
12) Thread[id=88, name=transaction-log-manager-0, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1166)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:883)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
13) Thread[id=37, name=NIOServerCxnFactory.SelectorThread-1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.select(NIOServerCnxnFactory.java:403)
at org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.run(NIOServerCnxnFactory.java:368)
14) Thread[id=36, name=NIOServerCxnFactory.SelectorThread-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.select(NIOServerCnxnFactory.java:403)
at org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.run(NIOServerCnxnFactory.java:368)
15) Thread[id=87, name=TxnMarkerSenderThread-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.IOUtil.write1(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.wakeup(WEPollSelectorImpl.java:235)
at java.base/java.nio.channels.spi.AbstractSelector$1.postInterrupt(AbstractSelector.java:111)
at java.base/java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:237)
at java.base/sun.nio.ch.SelectorImpl.begin(SelectorImpl.java:100)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:105)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:596)
at org.apache.kafka.server.util.InterBrokerSendThread.pollOnce(InterBrokerSendThread.java:110)
at org.apache.kafka.server.util.InterBrokerSendThread.doWork(InterBrokerSendThread.java:137)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
16) Thread[id=70, name=zk-broker-0-to-controller-alter-partition-channel-manager, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:596)
at org.apache.kafka.server.util.InterBrokerSendThread.pollOnce(InterBrokerSendThread.java:110)
at kafka.server.NodeToControllerRequestThread.doWork(NodeToControllerChannelManager.scala:377)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
17) Thread[id=505, name=h2sc-249-thread-8, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
18) Thread[id=68, name=data-plane-kafka-network-thread-0-ListenerName(PLAINTEXT)-PLAINTEXT-1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at kafka.network.Processor.poll(SocketServer.scala:1110)
at kafka.network.Processor.run(SocketServer.scala:1014)
at java.base/java.lang.Thread.run(Thread.java:1527)
19) Thread[id=501, name=h2sc-249-thread-4, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
20) Thread[id=686, name=zkCallback-251-thread-2, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:408)
at java.base/java.util.concurrent.LinkedTransferQueue$DualNode.await(LinkedTransferQueue.java:452)
at java.base/java.util.concurrent.SynchronousQueue$Transferer.xferLifo(SynchronousQueue.java:194)
at java.base/java.util.concurrent.SynchronousQueue.xfer(SynchronousQueue.java:235)
at java.base/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:338)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1015)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
21) Thread[id=490, name=Curator-Framework-0, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.DelayQueue.take(DelayQueue.java:243)
at java.base/java.util.concurrent.DelayQueue.take(DelayQueue.java:100)
at org.apache.curator.framework.imps.CuratorFrameworkImpl.backgroundOperationsLoop(CuratorFrameworkImpl.java:796)
at org.apache.curator.framework.imps.CuratorFrameworkImpl.access$300(CuratorFrameworkImpl.java:77)
at org.apache.curator.framework.imps.CuratorFrameworkImpl$4.call(CuratorFrameworkImpl.java:320)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:330)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:309)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1090)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
22) Thread[id=502, name=h2sc-249-thread-5, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
23) Thread[id=498, name=HttpClient@5840ded8-scheduler-1, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1166)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:883)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
24) Thread[id=609, name=h2sc-249-thread-9, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
25) Thread[id=50, name=NIOWorkerThread-1, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
26) Thread[id=482, name=h2sc-249-thread-1-processing-collection1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:182)
at org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:189)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:616)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:550)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produceTask(AdaptiveExecutionStrategy.java:526)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:251)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produce(AdaptiveExecutionStrategy.java:196)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:381)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1090)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
27) Thread[id=69, name=data-plane-kafka-network-thread-0-ListenerName(PLAINTEXT)-PLAINTEXT-2, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.IOUtil.write1(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.wakeup(WEPollSelectorImpl.java:235)
at java.base/java.nio.channels.spi.AbstractSelector$1.postInterrupt(AbstractSelector.java:111)
at java.base/java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:237)
at java.base/sun.nio.ch.SelectorImpl.begin(SelectorImpl.java:100)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:105)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at kafka.network.Processor.poll(SocketServer.scala:1110)
at kafka.network.Processor.run(SocketServer.scala:1014)
at java.base/java.lang.Thread.run(Thread.java:1527)
28) Thread[id=66, name=data-plane-kafka-socket-acceptor-ListenerName(PLAINTEXT)-PLAINTEXT-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at kafka.network.Acceptor.run(SocketServer.scala:694)
at java.base/java.lang.Thread.run(Thread.java:1527)
29) Thread[id=466, name=executor-Rebalance, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
30) Thread[id=455, name=CrossDCConsumer-242-thread-1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.poll(ConsumerNetworkClient.java:269)
at org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.poll(ConsumerNetworkClient.java:232)
at org.apache.kafka.clients.consumer.internals.AbstractCoordinator.ensureCoordinatorReady(AbstractCoordinator.java:290)
at org.apache.kafka.clients.consumer.internals.AbstractCoordinator.ensureCoordinatorReady(AbstractCoordinator.java:264)
at org.apache.kafka.clients.consumer.internals.ConsumerCoordinator.coordinatorUnknownAndUnreadySync(ConsumerCoordinator.java:451)
at org.apache.kafka.clients.consumer.internals.ConsumerCoordinator.poll(ConsumerCoordinator.java:483)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.updateAssignmentMetadataIfNeeded(ClassicKafkaConsumer.java:657)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:616)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.solr.crossdc.manager.consumer.KafkaCrossDcConsumer.pollAndProcessRequests(KafkaCrossDcConsumer.java:366)
at org.apache.solr.crossdc.manager.consumer.KafkaCrossDcConsumer.run(KafkaCrossDcConsumer.java:285)
at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:545)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:330)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:381)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1090)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
31) Thread[id=687, name=zkCallback-251-thread-3, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:408)
at java.base/java.util.concurrent.LinkedTransferQueue$DualNode.await(LinkedTransferQueue.java:452)
at java.base/java.util.concurrent.SynchronousQueue$Transferer.xferLifo(SynchronousQueue.java:194)
at java.base/java.util.concurrent.SynchronousQueue.xfer(SynchronousQueue.java:235)
at java.base/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:338)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1015)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
32) Thread[id=38, name=NIOServerCxnFactory.AcceptThread:/127.0.0.1:0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.IOUtil.drain(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.clearInterrupt(WEPollSelectorImpl.java:247)
at java.base/sun.nio.ch.WEPollSelectorImpl.processEvents(WEPollSelectorImpl.java:186)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:116)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.apache.zookeeper.server.NIOServerCnxnFactory$AcceptThread.select(NIOServerCnxnFactory.java:205)
at org.apache.zookeeper.server.NIOServerCnxnFactory$AcceptThread.run(NIOServerCnxnFactory.java:181)
33) Thread[id=484, name=h2sc-249-thread-2-processing-collection1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:182)
at org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:189)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:616)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:550)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produceTask(AdaptiveExecutionStrategy.java:526)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:251)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produce(AdaptiveExecutionStrategy.java:196)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:381)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1090)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
34) Thread[id=486, name=Curator-ConnectionStateManager-0, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:436)
at org.apache.curator.framework.state.ConnectionStateManager.processEvents(ConnectionStateManager.java:246)
at org.apache.curator.framework.state.ConnectionStateManager.lambda$start$0(ConnectionStateManager.java:127)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:330)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1090)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
35) Thread[id=53, name=SensorExpiryThread, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1166)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:883)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
36) Thread[id=48, name=SUITE-SolrAndKafkaReindexTest-seed#[77E68AE857775F98]-worker-SendThread(127.0.0.1:49533), state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/java.lang.Thread.sleepNanos0(Native Method)
at java.base/java.lang.Thread.sleepNanos(Thread.java:562)
at java.base/java.lang.Thread.sleep(Thread.java:593)
at org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:214)
at org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1374)
at org.apache.zookeeper.ClientCnxn$SendThread.cleanAndNotifyState(ClientCnxn.java:1342)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1310)
37) Thread[id=503, name=h2sc-249-thread-6, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
at __randomizedtesting.SeedInfo.seed([77E68AE857775F98]:0)
FAILED: org.apache.solr.crossdc.manager.SolrAndKafkaReindexTest.classMethod
Error Message:
java.lang.Exception: Error shutting down MiniSolrCloudCluster
Stack Trace:
java.lang.Exception: Error shutting down MiniSolrCloudCluster
at __randomizedtesting.SeedInfo.seed([77E68AE857775F98]:0)
at org.apache.solr.cloud.MiniSolrCloudCluster.checkForExceptions(MiniSolrCloudCluster.java:748)
at org.apache.solr.cloud.MiniSolrCloudCluster.shutdown(MiniSolrCloudCluster.java:669)
at org.apache.solr.crossdc.manager.SolrAndKafkaReindexTest.afterSolrAndKafkaIntegrationTest(SolrAndKafkaReindexTest.java:134)
at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
at java.base/java.lang.reflect.Method.invoke(Method.java:565)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1763)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:909)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
at org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
at java.base/java.lang.Thread.run(Thread.java:1527)
Suppressed: java.lang.RuntimeException: Timeout waiting for pool org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor@2f285479[Shutting down, pool size = 1, active threads = 1, queued tasks = 0, completed tasks = 4] to shutdown.
at org.apache.solr.common.util.ExecutorUtil.awaitTermination(ExecutorUtil.java:166)
at org.apache.solr.common.util.ExecutorUtil.awaitTermination(ExecutorUtil.java:153)
at org.apache.solr.common.util.ExecutorUtil.shutdownAndAwaitTermination(ExecutorUtil.java:119)
at org.apache.solr.core.CoreContainer.shutdown(CoreContainer.java:1325)
at org.apache.solr.servlet.CoreContainerProvider.close(CoreContainerProvider.java:132)
at org.apache.solr.servlet.CoreContainerProvider.contextDestroyed(CoreContainerProvider.java:90)
at org.eclipse.jetty.ee10.servlet.ServletContextHandler.callContextDestroyed(ServletContextHandler.java:1608)
at org.eclipse.jetty.ee10.servlet.ServletContextHandler.contextDestroyed(ServletContextHandler.java:529)
at org.eclipse.jetty.ee10.servlet.ServletHandler.doStop(ServletHandler.java:329)
at org.eclipse.jetty.util.component.AbstractLifeCycle.stop(AbstractLifeCycle.java:132)
at org.eclipse.jetty.util.component.ContainerLifeCycle.stop(ContainerLifeCycle.java:180)
at org.eclipse.jetty.util.component.ContainerLifeCycle.doStop(ContainerLifeCycle.java:202)
at org.eclipse.jetty.server.Handler$Abstract.doStop(Handler.java:499)
at org.eclipse.jetty.server.handler.ContextHandler.lambda$doStop$0(ContextHandler.java:841)
at org.eclipse.jetty.server.handler.ContextHandler$ScopedContext.call(ContextHandler.java:1488)
at org.eclipse.jetty.server.handler.ContextHandler.doStop(ContextHandler.java:841)
at org.eclipse.jetty.ee10.servlet.ServletContextHandler.stopContext(ServletContextHandler.java:1334)
at org.eclipse.jetty.ee10.servlet.ServletContextHandler.doStop(ServletContextHandler.java:1085)
at org.eclipse.jetty.util.component.AbstractLifeCycle.stop(AbstractLifeCycle.java:132)
at org.eclipse.jetty.util.component.ContainerLifeCycle.stop(ContainerLifeCycle.java:180)
at org.eclipse.jetty.util.component.ContainerLifeCycle.doStop(ContainerLifeCycle.java:202)
at org.eclipse.jetty.server.Handler$Abstract.doStop(Handler.java:499)
at org.eclipse.jetty.util.component.AbstractLifeCycle.stop(AbstractLifeCycle.java:132)
at org.eclipse.jetty.util.component.ContainerLifeCycle.stop(ContainerLifeCycle.java:180)
at org.eclipse.jetty.util.component.ContainerLifeCycle.doStop(ContainerLifeCycle.java:202)
at org.eclipse.jetty.server.Handler$Abstract.doStop(Handler.java:499)
at org.eclipse.jetty.util.component.AbstractLifeCycle.stop(AbstractLifeCycle.java:132)
at org.eclipse.jetty.util.component.ContainerLifeCycle.stop(ContainerLifeCycle.java:180)
at org.eclipse.jetty.util.component.ContainerLifeCycle.doStop(ContainerLifeCycle.java:202)
at org.eclipse.jetty.server.Handler$Abstract.doStop(Handler.java:499)
at org.eclipse.jetty.server.Server.doStop(Server.java:690)
at org.eclipse.jetty.util.component.AbstractLifeCycle.stop(AbstractLifeCycle.java:132)
at org.apache.solr.embedded.JettySolrRunner.stop(JettySolrRunner.java:586)
at org.apache.solr.cloud.MiniSolrCloudCluster.stopJettySolrRunner(MiniSolrCloudCluster.java:561)
at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$shutdown$1(MiniSolrCloudCluster.java:661)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:330)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:381)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1090)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
... 1 more
FAILED: org.apache.solr.crossdc.manager.SolrAndKafkaReindexTest.classMethod
Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 67 threads leaked from SUITE scope at org.apache.solr.crossdc.manager.SolrAndKafkaReindexTest:
1) Thread[id=98, name=Controller-0-to-broker-0-send-thread, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:242)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
2) Thread[id=79, name=LogDirFailureHandler, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:421)
at org.apache.kafka.storage.internals.log.LogDirFailureChannel.takeNextOfflineLogDir(LogDirFailureChannel.java:75)
at kafka.server.ReplicaManager$LogDirFailureHandler.doWork(ReplicaManager.scala:320)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
3) Thread[id=63, name=feature-zk-node-event-process-thread, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at kafka.server.FinalizedFeatureChangeListener$ChangeNotificationProcessorThread.doWork(FinalizedFeatureChangeListener.scala:154)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
4) Thread[id=74, name=ExpirationReaper-0-ElectLeader, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:292)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
5) Thread[id=92, name=data-plane-kafka-request-handler-3, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:436)
at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:484)
at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:113)
at java.base/java.lang.Thread.run(Thread.java:1527)
6) Thread[id=96, name=data-plane-kafka-request-handler-7, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:436)
at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:484)
at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:113)
at java.base/java.lang.Thread.run(Thread.java:1527)
7) Thread[id=50, name=NIOWorkerThread-2, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
8) Thread[id=56, name=ThrottledChannelReaper-ControllerMutation, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:292)
at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:178)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
9) Thread[id=625, name=h2sc-249-thread-9, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
10) Thread[id=87, name=transaction-log-manager-0, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1166)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:883)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
11) Thread[id=94, name=data-plane-kafka-request-handler-5, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:436)
at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:484)
at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:113)
at java.base/java.lang.Thread.run(Thread.java:1527)
12) Thread[id=49, name=NIOWorkerThread-1, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
13) Thread[id=83, name=ExpirationReaper-0-Heartbeat, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:301)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
14) Thread[id=93, name=data-plane-kafka-request-handler-4, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:436)
at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:484)
at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:113)
at java.base/java.lang.Thread.run(Thread.java:1527)
15) Thread[id=543, name=executor-Fetch, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
16) Thread[id=81, name=controller-event-thread, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
at kafka.controller.ControllerEventManager.kafka$controller$ControllerEventManager$$pollFromEventQueue(ControllerEventManager.scala:156)
at kafka.controller.ControllerEventManager$ControllerEventThread.doWork(ControllerEventManager.scala:129)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
17) Thread[id=75, name=ExpirationReaper-0-RemoteFetch, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:292)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
18) Thread[id=633, name=h2sc-249-thread-10, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
19) Thread[id=35, name=NIOServerCxnFactory.SelectorThread-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.select(NIOServerCnxnFactory.java:403)
at org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.run(NIOServerCnxnFactory.java:368)
20) Thread[id=47, name=SUITE-SolrAndKafkaReindexTest-seed#[77E68AE857775F98]-worker-SendThread(127.0.0.1:52637), state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1290)
21) Thread[id=39, name=SyncThread:0, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at org.apache.zookeeper.server.SyncRequestProcessor.run(SyncRequestProcessor.java:170)
22) Thread[id=51, name=NIOWorkerThread-3, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
23) Thread[id=65, name=data-plane-kafka-socket-acceptor-ListenerName(PLAINTEXT)-PLAINTEXT-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at kafka.network.Acceptor.acceptNewConnections(SocketServer.scala:745)
at kafka.network.Acceptor.run(SocketServer.scala:693)
at java.base/java.lang.Thread.run(Thread.java:1527)
24) Thread[id=698, name=zkCallback-251-thread-2, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:408)
at java.base/java.util.concurrent.LinkedTransferQueue$DualNode.await(LinkedTransferQueue.java:452)
at java.base/java.util.concurrent.SynchronousQueue$Transferer.xferLifo(SynchronousQueue.java:194)
at java.base/java.util.concurrent.SynchronousQueue.xfer(SynchronousQueue.java:235)
at java.base/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:338)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1015)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
25) Thread[id=40, name=ProcessThread(sid:0 cport:52637):, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:142)
26) Thread[id=69, name=zk-broker-0-to-controller-alter-partition-channel-manager, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:596)
at org.apache.kafka.server.util.InterBrokerSendThread.pollOnce(InterBrokerSendThread.java:110)
at kafka.server.NodeToControllerRequestThread.doWork(NodeToControllerChannelManager.scala:377)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
27) Thread[id=71, name=ExpirationReaper-0-Produce, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:292)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
28) Thread[id=95, name=data-plane-kafka-request-handler-6, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:436)
at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:484)
at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:113)
at java.base/java.lang.Thread.run(Thread.java:1527)
29) Thread[id=37, name=NIOServerCxnFactory.AcceptThread:/127.0.0.1:0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.apache.zookeeper.server.NIOServerCnxnFactory$AcceptThread.select(NIOServerCnxnFactory.java:205)
at org.apache.zookeeper.server.NIOServerCnxnFactory$AcceptThread.run(NIOServerCnxnFactory.java:181)
30) Thread[id=506, name=h2sc-249-thread-7, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
31) Thread[id=48, name=SUITE-SolrAndKafkaReindexTest-seed#[77E68AE857775F98]-worker-EventThread, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:537)
32) Thread[id=469, name=executor-Rebalance, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
33) Thread[id=55, name=ThrottledChannelReaper-Request, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:292)
at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:178)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
34) Thread[id=488, name=h2sc-249-thread-2-processing-collection1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:182)
at org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:189)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:616)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:550)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produceTask(AdaptiveExecutionStrategy.java:526)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:251)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produce(AdaptiveExecutionStrategy.java:196)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:381)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1090)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
35) Thread[id=637, name=zkCallback-251-thread-1, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:408)
at java.base/java.util.concurrent.LinkedTransferQueue$DualNode.await(LinkedTransferQueue.java:452)
at java.base/java.util.concurrent.SynchronousQueue$Transferer.xferLifo(SynchronousQueue.java:194)
at java.base/java.util.concurrent.SynchronousQueue.xfer(SynchronousQueue.java:235)
at java.base/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:338)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1015)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
36) Thread[id=73, name=ExpirationReaper-0-DeleteRecords, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:292)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
37) Thread[id=41, name=RequestThrottler, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at org.apache.zookeeper.server.RequestThrottler.run(RequestThrottler.java:147)
38) Thread[id=487, name=h2sc-249-thread-1-processing-collection1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:182)
at org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:189)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:616)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:550)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produceTask(AdaptiveExecutionStrategy.java:526)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:251)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produce(AdaptiveExecutionStrategy.java:196)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:381)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1090)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
39) Thread[id=66, name=data-plane-kafka-network-thread-0-ListenerName(PLAINTEXT)-PLAINTEXT-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at kafka.network.Processor.poll(SocketServer.scala:1110)
at kafka.network.Processor.run(SocketServer.scala:1014)
at java.base/java.lang.Thread.run(Thread.java:1527)
40) Thread[id=70, name=AddPartitionsToTxnSenderThread-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:596)
at org.apache.kafka.server.util.InterBrokerSendThread.pollOnce(InterBrokerSendThread.java:110)
at org.apache.kafka.server.util.InterBrokerSendThread.doWork(InterBrokerSendThread.java:137)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
41) Thread[id=53, name=ThrottledChannelReaper-Fetch, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:292)
at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:178)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
42) Thread[id=500, name=HttpClient@1bd44985-scheduler-1, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1166)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:883)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
43) Thread[id=82, name=ExpirationReaper-0-topic, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:292)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
44) Thread[id=67, name=data-plane-kafka-network-thread-0-ListenerName(PLAINTEXT)-PLAINTEXT-1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at kafka.network.Processor.poll(SocketServer.scala:1110)
at kafka.network.Processor.run(SocketServer.scala:1014)
at java.base/java.lang.Thread.run(Thread.java:1527)
45) Thread[id=97, name=/config/changes-event-process-thread, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at kafka.common.ZkNodeChangeNotificationListener$ChangeEventProcessThread.doWork(ZkNodeChangeNotificationListener.scala:147)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
46) Thread[id=88, name=ExpirationReaper-0-AlterAcls, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:292)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
47) Thread[id=503, name=h2sc-249-thread-4, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
48) Thread[id=72, name=ExpirationReaper-0-Fetch, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:306)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
49) Thread[id=85, name=group-metadata-manager-0, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1166)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:883)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
50) Thread[id=457, name=CrossDCConsumer-242-thread-1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:596)
at org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.poll(ConsumerNetworkClient.java:282)
at org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.poll(ConsumerNetworkClient.java:253)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:692)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.solr.crossdc.manager.consumer.KafkaCrossDcConsumer.pollAndProcessRequests(KafkaCrossDcConsumer.java:366)
at org.apache.solr.crossdc.manager.consumer.KafkaCrossDcConsumer.run(KafkaCrossDcConsumer.java:285)
at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:545)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:330)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:381)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1090)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
51) Thread[id=64, name=zk-broker-0-to-controller-forwarding-channel-manager, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:596)
at org.apache.kafka.server.util.InterBrokerSendThread.pollOnce(InterBrokerSendThread.java:110)
at kafka.server.NodeToControllerRequestThread.doWork(NodeToControllerChannelManager.scala:377)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
52) Thread[id=84, name=ExpirationReaper-0-Rebalance, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:292)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
53) Thread[id=490, name=Curator-ConnectionStateManager-0, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:436)
at org.apache.curator.framework.state.ConnectionStateManager.processEvents(ConnectionStateManager.java:246)
at org.apache.curator.framework.state.ConnectionStateManager.lambda$start$0(ConnectionStateManager.java:127)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:330)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1090)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
54) Thread[id=89, name=data-plane-kafka-request-handler-0, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:436)
at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:484)
at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:113)
at java.base/java.lang.Thread.run(Thread.java:1527)
55) Thread[id=91, name=data-plane-kafka-request-handler-2, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:436)
at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:484)
at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:113)
at java.base/java.lang.Thread.run(Thread.java:1527)
56) Thread[id=507, name=h2sc-249-thread-8, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
57) Thread[id=86, name=TxnMarkerSenderThread-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:596)
at org.apache.kafka.server.util.InterBrokerSendThread.pollOnce(InterBrokerSendThread.java:110)
at org.apache.kafka.server.util.InterBrokerSendThread.doWork(InterBrokerSendThread.java:137)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
58) Thread[id=504, name=h2sc-249-thread-5, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
59) Thread[id=505, name=h2sc-249-thread-6, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
60) Thread[id=68, name=data-plane-kafka-network-thread-0-ListenerName(PLAINTEXT)-PLAINTEXT-2, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at kafka.network.Processor.poll(SocketServer.scala:1110)
at kafka.network.Processor.run(SocketServer.scala:1014)
at java.base/java.lang.Thread.run(Thread.java:1527)
61) Thread[id=90, name=data-plane-kafka-request-handler-1, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:436)
at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:484)
at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:113)
at java.base/java.lang.Thread.run(Thread.java:1527)
62) Thread[id=36, name=NIOServerCxnFactory.SelectorThread-1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.select(NIOServerCnxnFactory.java:403)
at org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.run(NIOServerCnxnFactory.java:368)
63) Thread[id=52, name=SensorExpiryThread, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1166)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:883)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
64) Thread[id=54, name=ThrottledChannelReaper-Produce, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:292)
at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:178)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
65) Thread[id=494, name=Curator-Framework-0, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.DelayQueue.take(DelayQueue.java:243)
at java.base/java.util.concurrent.DelayQueue.take(DelayQueue.java:100)
at org.apache.curator.framework.imps.CuratorFrameworkImpl.backgroundOperationsLoop(CuratorFrameworkImpl.java:796)
at org.apache.curator.framework.imps.CuratorFrameworkImpl.access$300(CuratorFrameworkImpl.java:77)
at org.apache.curator.framework.imps.CuratorFrameworkImpl$4.call(CuratorFrameworkImpl.java:320)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:330)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:309)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1090)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
66) Thread[id=501, name=h2sc-249-thread-3, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
67) Thread[id=699, name=zkCallback-251-thread-3, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:408)
at java.base/java.util.concurrent.LinkedTransferQueue$DualNode.await(LinkedTransferQueue.java:452)
at java.base/java.util.concurrent.SynchronousQueue$Transferer.xferLifo(SynchronousQueue.java:194)
at java.base/java.util.concurrent.SynchronousQueue.xfer(SynchronousQueue.java:235)
at java.base/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:338)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1015)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 67 threads leaked from SUITE scope at org.apache.solr.crossdc.manager.SolrAndKafkaReindexTest:
1) Thread[id=98, name=Controller-0-to-broker-0-send-thread, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:242)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
2) Thread[id=79, name=LogDirFailureHandler, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:421)
at org.apache.kafka.storage.internals.log.LogDirFailureChannel.takeNextOfflineLogDir(LogDirFailureChannel.java:75)
at kafka.server.ReplicaManager$LogDirFailureHandler.doWork(ReplicaManager.scala:320)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
3) Thread[id=63, name=feature-zk-node-event-process-thread, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at kafka.server.FinalizedFeatureChangeListener$ChangeNotificationProcessorThread.doWork(FinalizedFeatureChangeListener.scala:154)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
4) Thread[id=74, name=ExpirationReaper-0-ElectLeader, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:292)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
5) Thread[id=92, name=data-plane-kafka-request-handler-3, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:436)
at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:484)
at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:113)
at java.base/java.lang.Thread.run(Thread.java:1527)
6) Thread[id=96, name=data-plane-kafka-request-handler-7, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:436)
at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:484)
at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:113)
at java.base/java.lang.Thread.run(Thread.java:1527)
7) Thread[id=50, name=NIOWorkerThread-2, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
8) Thread[id=56, name=ThrottledChannelReaper-ControllerMutation, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:292)
at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:178)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
9) Thread[id=625, name=h2sc-249-thread-9, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
10) Thread[id=87, name=transaction-log-manager-0, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1166)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:883)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
11) Thread[id=94, name=data-plane-kafka-request-handler-5, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:436)
at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:484)
at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:113)
at java.base/java.lang.Thread.run(Thread.java:1527)
12) Thread[id=49, name=NIOWorkerThread-1, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
13) Thread[id=83, name=ExpirationReaper-0-Heartbeat, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:301)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
14) Thread[id=93, name=data-plane-kafka-request-handler-4, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:436)
at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:484)
at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:113)
at java.base/java.lang.Thread.run(Thread.java:1527)
15) Thread[id=543, name=executor-Fetch, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
16) Thread[id=81, name=controller-event-thread, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
at kafka.controller.ControllerEventManager.kafka$controller$ControllerEventManager$$pollFromEventQueue(ControllerEventManager.scala:156)
at kafka.controller.ControllerEventManager$ControllerEventThread.doWork(ControllerEventManager.scala:129)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
17) Thread[id=75, name=ExpirationReaper-0-RemoteFetch, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:292)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
18) Thread[id=633, name=h2sc-249-thread-10, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
19) Thread[id=35, name=NIOServerCxnFactory.SelectorThread-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.select(NIOServerCnxnFactory.java:403)
at org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.run(NIOServerCnxnFactory.java:368)
20) Thread[id=47, name=SUITE-SolrAndKafkaReindexTest-seed#[77E68AE857775F98]-worker-SendThread(127.0.0.1:52637), state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1290)
21) Thread[id=39, name=SyncThread:0, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at org.apache.zookeeper.server.SyncRequestProcessor.run(SyncRequestProcessor.java:170)
22) Thread[id=51, name=NIOWorkerThread-3, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
23) Thread[id=65, name=data-plane-kafka-socket-acceptor-ListenerName(PLAINTEXT)-PLAINTEXT-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at kafka.network.Acceptor.acceptNewConnections(SocketServer.scala:745)
at kafka.network.Acceptor.run(SocketServer.scala:693)
at java.base/java.lang.Thread.run(Thread.java:1527)
24) Thread[id=698, name=zkCallback-251-thread-2, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:408)
at java.base/java.util.concurrent.LinkedTransferQueue$DualNode.await(LinkedTransferQueue.java:452)
at java.base/java.util.concurrent.SynchronousQueue$Transferer.xferLifo(SynchronousQueue.java:194)
at java.base/java.util.concurrent.SynchronousQueue.xfer(SynchronousQueue.java:235)
at java.base/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:338)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1015)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
25) Thread[id=40, name=ProcessThread(sid:0 cport:52637):, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:142)
26) Thread[id=69, name=zk-broker-0-to-controller-alter-partition-channel-manager, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:596)
at org.apache.kafka.server.util.InterBrokerSendThread.pollOnce(InterBrokerSendThread.java:110)
at kafka.server.NodeToControllerRequestThread.doWork(NodeToControllerChannelManager.scala:377)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
27) Thread[id=71, name=ExpirationReaper-0-Produce, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:292)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
28) Thread[id=95, name=data-plane-kafka-request-handler-6, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:436)
at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:484)
at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:113)
at java.base/java.lang.Thread.run(Thread.java:1527)
29) Thread[id=37, name=NIOServerCxnFactory.AcceptThread:/127.0.0.1:0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.apache.zookeeper.server.NIOServerCnxnFactory$AcceptThread.select(NIOServerCnxnFactory.java:205)
at org.apache.zookeeper.server.NIOServerCnxnFactory$AcceptThread.run(NIOServerCnxnFactory.java:181)
30) Thread[id=506, name=h2sc-249-thread-7, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
31) Thread[id=48, name=SUITE-SolrAndKafkaReindexTest-seed#[77E68AE857775F98]-worker-EventThread, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:537)
32) Thread[id=469, name=executor-Rebalance, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
33) Thread[id=55, name=ThrottledChannelReaper-Request, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:292)
at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:178)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
34) Thread[id=488, name=h2sc-249-thread-2-processing-collection1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:182)
at org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:189)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:616)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:550)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produceTask(AdaptiveExecutionStrategy.java:526)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:251)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produce(AdaptiveExecutionStrategy.java:196)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:381)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1090)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
35) Thread[id=637, name=zkCallback-251-thread-1, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:408)
at java.base/java.util.concurrent.LinkedTransferQueue$DualNode.await(LinkedTransferQueue.java:452)
at java.base/java.util.concurrent.SynchronousQueue$Transferer.xferLifo(SynchronousQueue.java:194)
at java.base/java.util.concurrent.SynchronousQueue.xfer(SynchronousQueue.java:235)
at java.base/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:338)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1015)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
36) Thread[id=73, name=ExpirationReaper-0-DeleteRecords, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:292)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
37) Thread[id=41, name=RequestThrottler, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at org.apache.zookeeper.server.RequestThrottler.run(RequestThrottler.java:147)
38) Thread[id=487, name=h2sc-249-thread-1-processing-collection1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:182)
at org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:189)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:616)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:550)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produceTask(AdaptiveExecutionStrategy.java:526)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:251)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produce(AdaptiveExecutionStrategy.java:196)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:381)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1090)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
39) Thread[id=66, name=data-plane-kafka-network-thread-0-ListenerName(PLAINTEXT)-PLAINTEXT-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at kafka.network.Processor.poll(SocketServer.scala:1110)
at kafka.network.Processor.run(SocketServer.scala:1014)
at java.base/java.lang.Thread.run(Thread.java:1527)
40) Thread[id=70, name=AddPartitionsToTxnSenderThread-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:596)
at org.apache.kafka.server.util.InterBrokerSendThread.pollOnce(InterBrokerSendThread.java:110)
at org.apache.kafka.server.util.InterBrokerSendThread.doWork(InterBrokerSendThread.java:137)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
41) Thread[id=53, name=ThrottledChannelReaper-Fetch, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:292)
at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:178)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
42) Thread[id=500, name=HttpClient@1bd44985-scheduler-1, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1166)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:883)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
43) Thread[id=82, name=ExpirationReaper-0-topic, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:292)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
44) Thread[id=67, name=data-plane-kafka-network-thread-0-ListenerName(PLAINTEXT)-PLAINTEXT-1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at kafka.network.Processor.poll(SocketServer.scala:1110)
at kafka.network.Processor.run(SocketServer.scala:1014)
at java.base/java.lang.Thread.run(Thread.java:1527)
45) Thread[id=97, name=/config/changes-event-process-thread, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at kafka.common.ZkNodeChangeNotificationListener$ChangeEventProcessThread.doWork(ZkNodeChangeNotificationListener.scala:147)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
46) Thread[id=88, name=ExpirationReaper-0-AlterAcls, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:292)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
47) Thread[id=503, name=h2sc-249-thread-4, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
48) Thread[id=72, name=ExpirationReaper-0-Fetch, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:306)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
49) Thread[id=85, name=group-metadata-manager-0, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1166)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:883)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
50) Thread[id=457, name=CrossDCConsumer-242-thread-1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:596)
at org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.poll(ConsumerNetworkClient.java:282)
at org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.poll(ConsumerNetworkClient.java:253)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:692)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.solr.crossdc.manager.consumer.KafkaCrossDcConsumer.pollAndProcessRequests(KafkaCrossDcConsumer.java:366)
at org.apache.solr.crossdc.manager.consumer.KafkaCrossDcConsumer.run(KafkaCrossDcConsumer.java:285)
at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:545)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:330)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:381)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1090)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
51) Thread[id=64, name=zk-broker-0-to-controller-forwarding-channel-manager, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:596)
at org.apache.kafka.server.util.InterBrokerSendThread.pollOnce(InterBrokerSendThread.java:110)
at kafka.server.NodeToControllerRequestThread.doWork(NodeToControllerChannelManager.scala:377)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
52) Thread[id=84, name=ExpirationReaper-0-Rebalance, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:292)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
53) Thread[id=490, name=Curator-ConnectionStateManager-0, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:436)
at org.apache.curator.framework.state.ConnectionStateManager.processEvents(ConnectionStateManager.java:246)
at org.apache.curator.framework.state.ConnectionStateManager.lambda$start$0(ConnectionStateManager.java:127)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:330)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1090)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
54) Thread[id=89, name=data-plane-kafka-request-handler-0, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:436)
at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:484)
at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:113)
at java.base/java.lang.Thread.run(Thread.java:1527)
55) Thread[id=91, name=data-plane-kafka-request-handler-2, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:436)
at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:484)
at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:113)
at java.base/java.lang.Thread.run(Thread.java:1527)
56) Thread[id=507, name=h2sc-249-thread-8, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
57) Thread[id=86, name=TxnMarkerSenderThread-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:596)
at org.apache.kafka.server.util.InterBrokerSendThread.pollOnce(InterBrokerSendThread.java:110)
at org.apache.kafka.server.util.InterBrokerSendThread.doWork(InterBrokerSendThread.java:137)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
58) Thread[id=504, name=h2sc-249-thread-5, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
59) Thread[id=505, name=h2sc-249-thread-6, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
60) Thread[id=68, name=data-plane-kafka-network-thread-0-ListenerName(PLAINTEXT)-PLAINTEXT-2, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at kafka.network.Processor.poll(SocketServer.scala:1110)
at kafka.network.Processor.run(SocketServer.scala:1014)
at java.base/java.lang.Thread.run(Thread.java:1527)
61) Thread[id=90, name=data-plane-kafka-request-handler-1, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:436)
at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:484)
at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:113)
at java.base/java.lang.Thread.run(Thread.java:1527)
62) Thread[id=36, name=NIOServerCxnFactory.SelectorThread-1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.select(NIOServerCnxnFactory.java:403)
at org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.run(NIOServerCnxnFactory.java:368)
63) Thread[id=52, name=SensorExpiryThread, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1166)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:883)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
64) Thread[id=54, name=ThrottledChannelReaper-Produce, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:292)
at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:178)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
65) Thread[id=494, name=Curator-Framework-0, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.DelayQueue.take(DelayQueue.java:243)
at java.base/java.util.concurrent.DelayQueue.take(DelayQueue.java:100)
at org.apache.curator.framework.imps.CuratorFrameworkImpl.backgroundOperationsLoop(CuratorFrameworkImpl.java:796)
at org.apache.curator.framework.imps.CuratorFrameworkImpl.access$300(CuratorFrameworkImpl.java:77)
at org.apache.curator.framework.imps.CuratorFrameworkImpl$4.call(CuratorFrameworkImpl.java:320)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:330)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:309)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1090)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
66) Thread[id=501, name=h2sc-249-thread-3, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
67) Thread[id=699, name=zkCallback-251-thread-3, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:408)
at java.base/java.util.concurrent.LinkedTransferQueue$DualNode.await(LinkedTransferQueue.java:452)
at java.base/java.util.concurrent.SynchronousQueue$Transferer.xferLifo(SynchronousQueue.java:194)
at java.base/java.util.concurrent.SynchronousQueue.xfer(SynchronousQueue.java:235)
at java.base/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:338)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1015)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
at __randomizedtesting.SeedInfo.seed([77E68AE857775F98]:0)
FAILED: org.apache.solr.crossdc.manager.SolrAndKafkaReindexTest.classMethod
Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=66, name=data-plane-kafka-network-thread-0-ListenerName(PLAINTEXT)-PLAINTEXT-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.IOUtil.write1(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.wakeup(WEPollSelectorImpl.java:235)
at java.base/java.nio.channels.spi.AbstractSelector$1.postInterrupt(AbstractSelector.java:111)
at java.base/java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:237)
at java.base/sun.nio.ch.SelectorImpl.begin(SelectorImpl.java:100)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:105)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at kafka.network.Processor.poll(SocketServer.scala:1110)
at kafka.network.Processor.run(SocketServer.scala:1014)
at java.base/java.lang.Thread.run(Thread.java:1527)
2) Thread[id=70, name=AddPartitionsToTxnSenderThread-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at org.apache.kafka.server.util.InterBrokerSendThread.drainGeneratedRequests(InterBrokerSendThread.java:90)
at org.apache.kafka.server.util.InterBrokerSendThread.pollOnce(InterBrokerSendThread.java:107)
at org.apache.kafka.server.util.InterBrokerSendThread.doWork(InterBrokerSendThread.java:137)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
3) Thread[id=50, name=NIOWorkerThread-2, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
4) Thread[id=625, name=h2sc-249-thread-9, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
5) Thread[id=87, name=transaction-log-manager-0, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1166)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:883)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
6) Thread[id=49, name=NIOWorkerThread-1, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
7) Thread[id=500, name=HttpClient@1bd44985-scheduler-1, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1166)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:883)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
8) Thread[id=67, name=data-plane-kafka-network-thread-0-ListenerName(PLAINTEXT)-PLAINTEXT-1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at kafka.network.Processor.poll(SocketServer.scala:1110)
at kafka.network.Processor.run(SocketServer.scala:1014)
at java.base/java.lang.Thread.run(Thread.java:1527)
9) Thread[id=543, name=executor-Fetch, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
10) Thread[id=633, name=h2sc-249-thread-10, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
11) Thread[id=503, name=h2sc-249-thread-4, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
12) Thread[id=35, name=NIOServerCxnFactory.SelectorThread-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/java.lang.Thread.holdsLock(Native Method)
at java.base/sun.nio.ch.SelectorImpl.processDeregisterQueue(SelectorImpl.java:264)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:103)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.select(NIOServerCnxnFactory.java:403)
at org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.run(NIOServerCnxnFactory.java:368)
13) Thread[id=85, name=group-metadata-manager-0, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1166)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:883)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
14) Thread[id=457, name=CrossDCConsumer-242-thread-1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/java.lang.StackTraceElement.initStackTraceElements(Native Method)
at java.base/java.lang.StackTraceElement.of(StackTraceElement.java:563)
at java.base/java.lang.Throwable.getOurStackTrace(Throwable.java:866)
at java.base/java.lang.Throwable.getStackTrace(Throwable.java:858)
at org.apache.logging.log4j.core.pattern.ThrowableStackTraceRenderer$Context$Metadata.populateMetadata(ThrowableStackTraceRenderer.java:291)
at org.apache.logging.log4j.core.pattern.ThrowableStackTraceRenderer$Context$Metadata.ofThrowable(ThrowableStackTraceRenderer.java:278)
at org.apache.logging.log4j.core.pattern.ThrowableStackTraceRenderer.createContext(ThrowableStackTraceRenderer.java:67)
at org.apache.logging.log4j.core.pattern.ThrowableStackTraceRenderer.renderThrowable(ThrowableStackTraceRenderer.java:55)
at org.apache.logging.log4j.core.pattern.ThrowablePatternConverter.format(ThrowablePatternConverter.java:130)
at org.apache.logging.log4j.core.layout.PatternLayout$NoFormatPatternSerializer.toSerializable(PatternLayout.java:354)
at org.apache.logging.log4j.core.layout.PatternLayout.toText(PatternLayout.java:251)
at org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:237)
at org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:57)
at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:227)
at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:220)
at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:211)
at org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:160)
at org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:133)
at org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:124)
at org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:88)
at org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:711)
at org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:669)
at org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:645)
at org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:589)
at org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:92)
at org.apache.logging.log4j.core.Logger.log(Logger.java:187)
at org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2970)
at org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2922)
at org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2904)
at org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2648)
at org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:2587)
at org.apache.logging.slf4j.Log4jLogger.error(Log4jLogger.java:313)
at org.apache.solr.crossdc.manager.consumer.KafkaCrossDcConsumer.pollAndProcessRequests(KafkaCrossDcConsumer.java:540)
at org.apache.solr.crossdc.manager.consumer.KafkaCrossDcConsumer.run(KafkaCrossDcConsumer.java:285)
at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:545)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:330)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:381)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1090)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
15) Thread[id=47, name=SUITE-SolrAndKafkaReindexTest-seed#[77E68AE857775F98]-worker-SendThread(127.0.0.1:52637), state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/java.lang.Thread.sleepNanos0(Native Method)
at java.base/java.lang.Thread.sleepNanos(Thread.java:562)
at java.base/java.lang.Thread.sleep(Thread.java:593)
at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:362)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1197)
16) Thread[id=51, name=NIOWorkerThread-3, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
17) Thread[id=64, name=zk-broker-0-to-controller-forwarding-channel-manager, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:596)
at org.apache.kafka.server.util.InterBrokerSendThread.pollOnce(InterBrokerSendThread.java:110)
at kafka.server.NodeToControllerRequestThread.doWork(NodeToControllerChannelManager.scala:377)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
18) Thread[id=490, name=Curator-ConnectionStateManager-0, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:436)
at org.apache.curator.framework.state.ConnectionStateManager.processEvents(ConnectionStateManager.java:246)
at org.apache.curator.framework.state.ConnectionStateManager.lambda$start$0(ConnectionStateManager.java:127)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:330)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1090)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
19) Thread[id=65, name=data-plane-kafka-socket-acceptor-ListenerName(PLAINTEXT)-PLAINTEXT-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.IOUtil.write1(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.wakeup(WEPollSelectorImpl.java:235)
at java.base/java.nio.channels.spi.AbstractSelector$1.postInterrupt(AbstractSelector.java:111)
at java.base/java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:237)
at java.base/sun.nio.ch.SelectorImpl.begin(SelectorImpl.java:100)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:105)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at kafka.network.Acceptor.acceptNewConnections(SocketServer.scala:745)
at kafka.network.Acceptor.run(SocketServer.scala:693)
at java.base/java.lang.Thread.run(Thread.java:1527)
20) Thread[id=698, name=zkCallback-251-thread-2, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:408)
at java.base/java.util.concurrent.LinkedTransferQueue$DualNode.await(LinkedTransferQueue.java:452)
at java.base/java.util.concurrent.SynchronousQueue$Transferer.xferLifo(SynchronousQueue.java:194)
at java.base/java.util.concurrent.SynchronousQueue.xfer(SynchronousQueue.java:235)
at java.base/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:338)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1015)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
21) Thread[id=507, name=h2sc-249-thread-8, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
22) Thread[id=69, name=zk-broker-0-to-controller-alter-partition-channel-manager, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:596)
at org.apache.kafka.server.util.InterBrokerSendThread.pollOnce(InterBrokerSendThread.java:110)
at kafka.server.NodeToControllerRequestThread.doWork(NodeToControllerChannelManager.scala:377)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
23) Thread[id=86, name=TxnMarkerSenderThread-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.IOUtil.drain(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.clearInterrupt(WEPollSelectorImpl.java:247)
at java.base/sun.nio.ch.WEPollSelectorImpl.processEvents(WEPollSelectorImpl.java:186)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:116)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:596)
at org.apache.kafka.server.util.InterBrokerSendThread.pollOnce(InterBrokerSendThread.java:110)
at org.apache.kafka.server.util.InterBrokerSendThread.doWork(InterBrokerSendThread.java:137)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
24) Thread[id=37, name=NIOServerCxnFactory.AcceptThread:/127.0.0.1:0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.apache.zookeeper.server.NIOServerCnxnFactory$AcceptThread.select(NIOServerCnxnFactory.java:205)
at org.apache.zookeeper.server.NIOServerCnxnFactory$AcceptThread.run(NIOServerCnxnFactory.java:181)
25) Thread[id=506, name=h2sc-249-thread-7, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
26) Thread[id=469, name=executor-Rebalance, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
27) Thread[id=504, name=h2sc-249-thread-5, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
28) Thread[id=505, name=h2sc-249-thread-6, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
29) Thread[id=488, name=h2sc-249-thread-2-processing-collection1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:182)
at org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:189)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:616)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:550)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produceTask(AdaptiveExecutionStrategy.java:526)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:251)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produce(AdaptiveExecutionStrategy.java:196)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:381)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1090)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
30) Thread[id=68, name=data-plane-kafka-network-thread-0-ListenerName(PLAINTEXT)-PLAINTEXT-2, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at kafka.network.Processor.poll(SocketServer.scala:1110)
at kafka.network.Processor.run(SocketServer.scala:1014)
at java.base/java.lang.Thread.run(Thread.java:1527)
31) Thread[id=637, name=zkCallback-251-thread-1, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:408)
at java.base/java.util.concurrent.LinkedTransferQueue$DualNode.await(LinkedTransferQueue.java:452)
at java.base/java.util.concurrent.SynchronousQueue$Transferer.xferLifo(SynchronousQueue.java:194)
at java.base/java.util.concurrent.SynchronousQueue.xfer(SynchronousQueue.java:235)
at java.base/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:338)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1015)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
32) Thread[id=36, name=NIOServerCxnFactory.SelectorThread-1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.select(NIOServerCnxnFactory.java:403)
at org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.run(NIOServerCnxnFactory.java:368)
33) Thread[id=52, name=SensorExpiryThread, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1166)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:883)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
34) Thread[id=494, name=Curator-Framework-0, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.DelayQueue.take(DelayQueue.java:243)
at java.base/java.util.concurrent.DelayQueue.take(DelayQueue.java:100)
at org.apache.curator.framework.imps.CuratorFrameworkImpl.backgroundOperationsLoop(CuratorFrameworkImpl.java:796)
at org.apache.curator.framework.imps.CuratorFrameworkImpl.access$300(CuratorFrameworkImpl.java:77)
at org.apache.curator.framework.imps.CuratorFrameworkImpl$4.call(CuratorFrameworkImpl.java:320)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:330)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:309)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1090)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
35) Thread[id=487, name=h2sc-249-thread-1-processing-collection1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:182)
at org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:189)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:616)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:550)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produceTask(AdaptiveExecutionStrategy.java:526)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:251)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produce(AdaptiveExecutionStrategy.java:196)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:381)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1090)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
36) Thread[id=501, name=h2sc-249-thread-3, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
37) Thread[id=699, name=zkCallback-251-thread-3, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:408)
at java.base/java.util.concurrent.LinkedTransferQueue$DualNode.await(LinkedTransferQueue.java:452)
at java.base/java.util.concurrent.SynchronousQueue$Transferer.xferLifo(SynchronousQueue.java:194)
at java.base/java.util.concurrent.SynchronousQueue.xfer(SynchronousQueue.java:235)
at java.base/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:338)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1015)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=66, name=data-plane-kafka-network-thread-0-ListenerName(PLAINTEXT)-PLAINTEXT-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.IOUtil.write1(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.wakeup(WEPollSelectorImpl.java:235)
at java.base/java.nio.channels.spi.AbstractSelector$1.postInterrupt(AbstractSelector.java:111)
at java.base/java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:237)
at java.base/sun.nio.ch.SelectorImpl.begin(SelectorImpl.java:100)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:105)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at kafka.network.Processor.poll(SocketServer.scala:1110)
at kafka.network.Processor.run(SocketServer.scala:1014)
at java.base/java.lang.Thread.run(Thread.java:1527)
2) Thread[id=70, name=AddPartitionsToTxnSenderThread-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at org.apache.kafka.server.util.InterBrokerSendThread.drainGeneratedRequests(InterBrokerSendThread.java:90)
at org.apache.kafka.server.util.InterBrokerSendThread.pollOnce(InterBrokerSendThread.java:107)
at org.apache.kafka.server.util.InterBrokerSendThread.doWork(InterBrokerSendThread.java:137)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
3) Thread[id=50, name=NIOWorkerThread-2, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
4) Thread[id=625, name=h2sc-249-thread-9, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
5) Thread[id=87, name=transaction-log-manager-0, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1166)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:883)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
6) Thread[id=49, name=NIOWorkerThread-1, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
7) Thread[id=500, name=HttpClient@1bd44985-scheduler-1, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1166)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:883)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
8) Thread[id=67, name=data-plane-kafka-network-thread-0-ListenerName(PLAINTEXT)-PLAINTEXT-1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at kafka.network.Processor.poll(SocketServer.scala:1110)
at kafka.network.Processor.run(SocketServer.scala:1014)
at java.base/java.lang.Thread.run(Thread.java:1527)
9) Thread[id=543, name=executor-Fetch, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
10) Thread[id=633, name=h2sc-249-thread-10, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
11) Thread[id=503, name=h2sc-249-thread-4, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
12) Thread[id=35, name=NIOServerCxnFactory.SelectorThread-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/java.lang.Thread.holdsLock(Native Method)
at java.base/sun.nio.ch.SelectorImpl.processDeregisterQueue(SelectorImpl.java:264)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:103)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.select(NIOServerCnxnFactory.java:403)
at org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.run(NIOServerCnxnFactory.java:368)
13) Thread[id=85, name=group-metadata-manager-0, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1166)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:883)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
14) Thread[id=457, name=CrossDCConsumer-242-thread-1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/java.lang.StackTraceElement.initStackTraceElements(Native Method)
at java.base/java.lang.StackTraceElement.of(StackTraceElement.java:563)
at java.base/java.lang.Throwable.getOurStackTrace(Throwable.java:866)
at java.base/java.lang.Throwable.getStackTrace(Throwable.java:858)
at org.apache.logging.log4j.core.pattern.ThrowableStackTraceRenderer$Context$Metadata.populateMetadata(ThrowableStackTraceRenderer.java:291)
at org.apache.logging.log4j.core.pattern.ThrowableStackTraceRenderer$Context$Metadata.ofThrowable(ThrowableStackTraceRenderer.java:278)
at org.apache.logging.log4j.core.pattern.ThrowableStackTraceRenderer.createContext(ThrowableStackTraceRenderer.java:67)
at org.apache.logging.log4j.core.pattern.ThrowableStackTraceRenderer.renderThrowable(ThrowableStackTraceRenderer.java:55)
at org.apache.logging.log4j.core.pattern.ThrowablePatternConverter.format(ThrowablePatternConverter.java:130)
at org.apache.logging.log4j.core.layout.PatternLayout$NoFormatPatternSerializer.toSerializable(PatternLayout.java:354)
at org.apache.logging.log4j.core.layout.PatternLayout.toText(PatternLayout.java:251)
at org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:237)
at org.apache.logging.log4j.core.layout.PatternLayout.encode(PatternLayout.java:57)
at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:227)
at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:220)
at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:211)
at org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:160)
at org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:133)
at org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:124)
at org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:88)
at org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:711)
at org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:669)
at org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:645)
at org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:589)
at org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:92)
at org.apache.logging.log4j.core.Logger.log(Logger.java:187)
at org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2970)
at org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2922)
at org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2904)
at org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2648)
at org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:2587)
at org.apache.logging.slf4j.Log4jLogger.error(Log4jLogger.java:313)
at org.apache.solr.crossdc.manager.consumer.KafkaCrossDcConsumer.pollAndProcessRequests(KafkaCrossDcConsumer.java:540)
at org.apache.solr.crossdc.manager.consumer.KafkaCrossDcConsumer.run(KafkaCrossDcConsumer.java:285)
at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:545)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:330)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:381)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1090)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
15) Thread[id=47, name=SUITE-SolrAndKafkaReindexTest-seed#[77E68AE857775F98]-worker-SendThread(127.0.0.1:52637), state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/java.lang.Thread.sleepNanos0(Native Method)
at java.base/java.lang.Thread.sleepNanos(Thread.java:562)
at java.base/java.lang.Thread.sleep(Thread.java:593)
at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:362)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1197)
16) Thread[id=51, name=NIOWorkerThread-3, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
17) Thread[id=64, name=zk-broker-0-to-controller-forwarding-channel-manager, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:596)
at org.apache.kafka.server.util.InterBrokerSendThread.pollOnce(InterBrokerSendThread.java:110)
at kafka.server.NodeToControllerRequestThread.doWork(NodeToControllerChannelManager.scala:377)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
18) Thread[id=490, name=Curator-ConnectionStateManager-0, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:436)
at org.apache.curator.framework.state.ConnectionStateManager.processEvents(ConnectionStateManager.java:246)
at org.apache.curator.framework.state.ConnectionStateManager.lambda$start$0(ConnectionStateManager.java:127)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:330)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1090)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
19) Thread[id=65, name=data-plane-kafka-socket-acceptor-ListenerName(PLAINTEXT)-PLAINTEXT-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.IOUtil.write1(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.wakeup(WEPollSelectorImpl.java:235)
at java.base/java.nio.channels.spi.AbstractSelector$1.postInterrupt(AbstractSelector.java:111)
at java.base/java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:237)
at java.base/sun.nio.ch.SelectorImpl.begin(SelectorImpl.java:100)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:105)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at kafka.network.Acceptor.acceptNewConnections(SocketServer.scala:745)
at kafka.network.Acceptor.run(SocketServer.scala:693)
at java.base/java.lang.Thread.run(Thread.java:1527)
20) Thread[id=698, name=zkCallback-251-thread-2, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:408)
at java.base/java.util.concurrent.LinkedTransferQueue$DualNode.await(LinkedTransferQueue.java:452)
at java.base/java.util.concurrent.SynchronousQueue$Transferer.xferLifo(SynchronousQueue.java:194)
at java.base/java.util.concurrent.SynchronousQueue.xfer(SynchronousQueue.java:235)
at java.base/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:338)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1015)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
21) Thread[id=507, name=h2sc-249-thread-8, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
22) Thread[id=69, name=zk-broker-0-to-controller-alter-partition-channel-manager, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:596)
at org.apache.kafka.server.util.InterBrokerSendThread.pollOnce(InterBrokerSendThread.java:110)
at kafka.server.NodeToControllerRequestThread.doWork(NodeToControllerChannelManager.scala:377)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
23) Thread[id=86, name=TxnMarkerSenderThread-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.IOUtil.drain(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.clearInterrupt(WEPollSelectorImpl.java:247)
at java.base/sun.nio.ch.WEPollSelectorImpl.processEvents(WEPollSelectorImpl.java:186)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:116)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:596)
at org.apache.kafka.server.util.InterBrokerSendThread.pollOnce(InterBrokerSendThread.java:110)
at org.apache.kafka.server.util.InterBrokerSendThread.doWork(InterBrokerSendThread.java:137)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
24) Thread[id=37, name=NIOServerCxnFactory.AcceptThread:/127.0.0.1:0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.apache.zookeeper.server.NIOServerCnxnFactory$AcceptThread.select(NIOServerCnxnFactory.java:205)
at org.apache.zookeeper.server.NIOServerCnxnFactory$AcceptThread.run(NIOServerCnxnFactory.java:181)
25) Thread[id=506, name=h2sc-249-thread-7, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
26) Thread[id=469, name=executor-Rebalance, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
27) Thread[id=504, name=h2sc-249-thread-5, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
28) Thread[id=505, name=h2sc-249-thread-6, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
29) Thread[id=488, name=h2sc-249-thread-2-processing-collection1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:182)
at org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:189)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:616)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:550)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produceTask(AdaptiveExecutionStrategy.java:526)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:251)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produce(AdaptiveExecutionStrategy.java:196)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:381)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1090)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
30) Thread[id=68, name=data-plane-kafka-network-thread-0-ListenerName(PLAINTEXT)-PLAINTEXT-2, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at kafka.network.Processor.poll(SocketServer.scala:1110)
at kafka.network.Processor.run(SocketServer.scala:1014)
at java.base/java.lang.Thread.run(Thread.java:1527)
31) Thread[id=637, name=zkCallback-251-thread-1, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:408)
at java.base/java.util.concurrent.LinkedTransferQueue$DualNode.await(LinkedTransferQueue.java:452)
at java.base/java.util.concurrent.SynchronousQueue$Transferer.xferLifo(SynchronousQueue.java:194)
at java.base/java.util.concurrent.SynchronousQueue.xfer(SynchronousQueue.java:235)
at java.base/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:338)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1015)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
32) Thread[id=36, name=NIOServerCxnFactory.SelectorThread-1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.select(NIOServerCnxnFactory.java:403)
at org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.run(NIOServerCnxnFactory.java:368)
33) Thread[id=52, name=SensorExpiryThread, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:271)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1803)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1166)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:883)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
34) Thread[id=494, name=Curator-Framework-0, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at java.base/java.util.concurrent.DelayQueue.take(DelayQueue.java:243)
at java.base/java.util.concurrent.DelayQueue.take(DelayQueue.java:100)
at org.apache.curator.framework.imps.CuratorFrameworkImpl.backgroundOperationsLoop(CuratorFrameworkImpl.java:796)
at org.apache.curator.framework.imps.CuratorFrameworkImpl.access$300(CuratorFrameworkImpl.java:77)
at org.apache.curator.framework.imps.CuratorFrameworkImpl$4.call(CuratorFrameworkImpl.java:320)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:330)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:309)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1090)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
35) Thread[id=487, name=h2sc-249-thread-1-processing-collection1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.WEPoll.wait(Native Method)
at java.base/sun.nio.ch.WEPollSelectorImpl.doSelect(WEPollSelectorImpl.java:108)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:182)
at org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:189)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:616)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:550)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produceTask(AdaptiveExecutionStrategy.java:526)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:251)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produce(AdaptiveExecutionStrategy.java:196)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:381)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1090)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
36) Thread[id=501, name=h2sc-249-thread-3, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:369)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:520)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4367)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:4313)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1752)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1016)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
37) Thread[id=699, name=zkCallback-251-thread-3, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:408)
at java.base/java.util.concurrent.LinkedTransferQueue$DualNode.await(LinkedTransferQueue.java:452)
at java.base/java.util.concurrent.SynchronousQueue$Transferer.xferLifo(SynchronousQueue.java:194)
at java.base/java.util.concurrent.SynchronousQueue.xfer(SynchronousQueue.java:235)
at java.base/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:338)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1015)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1076)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:614)
at java.base/java.lang.Thread.run(Thread.java:1527)
at __randomizedtesting.SeedInfo.seed([77E68AE857775F98]:0)
[JENKINS] Solr-main-Windows (64bit/hotspot/jdk-26) - Build # 8959 - Still Failing!
Posted by Policeman Jenkins Server via builds <bu...@solr.apache.org>.
Build: https://jenkins.thetaphi.de/job/Solr-main-Windows/8959/
Java: 64bit/hotspot/jdk-26 -XX:-UseCompressedOops -XX:+UseG1GC
All tests passed
[JENKINS] Solr-main-Windows (64bit/hotspot/jdk-23.0.2) - Build # 8958 - Still Failing!
Posted by Policeman Jenkins Server via builds <bu...@solr.apache.org>.
Build: https://jenkins.thetaphi.de/job/Solr-main-Windows/8958/
Java: 64bit/hotspot/jdk-23.0.2 -XX:+UseCompressedOops -XX:+UseG1GC
All tests passed