You are viewing a plain text version of this content. The canonical link for it is here.
Posted to builds@solr.apache.org by Policeman Jenkins Server via builds <bu...@solr.apache.org> on 2026/05/24 23:25:38 UTC
[JENKINS] Solr-main-Linux (64bit/hotspot/jdk-26) - Build # 33529 - Still Unstable!
Build: https://jenkins.thetaphi.de/job/Solr-main-Linux/33529/
Java: 64bit/hotspot/jdk-26 -XX:+UseCompressedOops -XX:+UseG1GC
1 tests failed.
FAILED: org.apache.solr.handler.component.AsyncTrackerSemaphoreLeakTest.testSemaphoreLeakOnLBRetry
Error Message:
java.lang.AssertionError: BUG (LBJettySolrClient retry deadlock): futures did not complete within 30s. IO threads are permanently blocked in semaphore.acquire() because the retry fires synchronously on the IO thread before onComplete can release().
Stack Trace:
java.lang.AssertionError: BUG (LBJettySolrClient retry deadlock): futures did not complete within 30s. IO threads are permanently blocked in semaphore.acquire() because the retry fires synchronously on the IO thread before onComplete can release().
at __randomizedtesting.SeedInfo.seed([6B3ED28A413C4CDB:B89E29612B541459]:0)
at org.junit.Assert.fail(Assert.java:89)
at org.apache.solr.handler.component.AsyncTrackerSemaphoreLeakTest.testSemaphoreLeakOnLBRetry(AsyncTrackerSemaphoreLeakTest.java:201)
at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
at java.base/java.lang.reflect.Method.invoke(Method.java:565)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1763)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48)
at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
at org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
at org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
at java.base/java.lang.Thread.run(Thread.java:1516)
[JENKINS] Solr-main-Linux (64bit/hotspot/jdk-23.0.2) - Build # 33532 - Still Unstable!
Posted by Policeman Jenkins Server via builds <bu...@solr.apache.org>.
Build: https://jenkins.thetaphi.de/job/Solr-main-Linux/33532/
Java: 64bit/hotspot/jdk-23.0.2 -XX:+UseCompressedOops -XX:+UseSerialGC
3 tests failed.
FAILED: org.apache.solr.crossdc.manager.SolrAndKafkaReindexTest.classMethod
Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=67, name=AddPartitionsToTxnSenderThread-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:596)
at org.apache.kafka.server.util.InterBrokerSendThread.pollOnce(InterBrokerSendThread.java:110)
at org.apache.kafka.server.util.InterBrokerSendThread.doWork(InterBrokerSendThread.java:137)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
2) Thread[id=82, name=group-metadata-manager-0, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1763)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
3) Thread[id=641, name=h2sc-262-thread-10, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
4) Thread[id=524, name=h2sc-262-thread-5, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
5) Thread[id=526, name=h2sc-262-thread-7, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
6) Thread[id=522, name=h2sc-262-thread-3, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
7) Thread[id=521, name=HttpClient@1f48b295-scheduler-1, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1763)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
8) Thread[id=527, name=h2sc-262-thread-8, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
9) Thread[id=518, name=Curator-Framework-0, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at java.base/java.util.concurrent.DelayQueue.take(DelayQueue.java:243)
at java.base/java.util.concurrent.DelayQueue.take(DelayQueue.java:100)
at org.apache.curator.framework.imps.CuratorFrameworkImpl.backgroundOperationsLoop(CuratorFrameworkImpl.java:796)
at org.apache.curator.framework.imps.CuratorFrameworkImpl.access$300(CuratorFrameworkImpl.java:77)
at org.apache.curator.framework.imps.CuratorFrameworkImpl$4.call(CuratorFrameworkImpl.java:320)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
10) Thread[id=33, name=NIOServerCxnFactory.SelectorThread-1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.IOUtil.drain(Native Method)
at java.base/sun.nio.ch.EventFD.reset(EventFD.java:52)
at java.base/sun.nio.ch.EPollSelectorImpl.clearInterrupt(EPollSelectorImpl.java:269)
at java.base/sun.nio.ch.EPollSelectorImpl.processEvents(EPollSelectorImpl.java:207)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:141)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.select(NIOServerCnxnFactory.java:403)
at org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.run(NIOServerCnxnFactory.java:368)
11) Thread[id=65, name=data-plane-kafka-network-thread-0-ListenerName(PLAINTEXT)-PLAINTEXT-2, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EventFD.set0(Native Method)
at java.base/sun.nio.ch.EventFD.set(EventFD.java:48)
at java.base/sun.nio.ch.EPollSelectorImpl.wakeup(EPollSelectorImpl.java:257)
at java.base/java.nio.channels.spi.AbstractSelector$1.postInterrupt(AbstractSelector.java:115)
at java.base/java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:241)
at java.base/sun.nio.ch.SelectorImpl.begin(SelectorImpl.java:100)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:115)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at kafka.network.Processor.poll(SocketServer.scala:1110)
at kafka.network.Processor.run(SocketServer.scala:1014)
at java.base/java.lang.Thread.run(Thread.java:1575)
12) Thread[id=693, name=zkCallback-264-thread-3, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:410)
at java.base/java.util.concurrent.LinkedTransferQueue$DualNode.await(LinkedTransferQueue.java:452)
at java.base/java.util.concurrent.SynchronousQueue$Transferer.xferLifo(SynchronousQueue.java:194)
at java.base/java.util.concurrent.SynchronousQueue.xfer(SynchronousQueue.java:235)
at java.base/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:338)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1069)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
13) Thread[id=515, name=Curator-ConnectionStateManager-0, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1763)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:435)
at org.apache.curator.framework.state.ConnectionStateManager.processEvents(ConnectionStateManager.java:246)
at org.apache.curator.framework.state.ConnectionStateManager.lambda$start$0(ConnectionStateManager.java:127)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
14) Thread[id=692, name=zkCallback-264-thread-2, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:410)
at java.base/java.util.concurrent.LinkedTransferQueue$DualNode.await(LinkedTransferQueue.java:452)
at java.base/java.util.concurrent.SynchronousQueue$Transferer.xferLifo(SynchronousQueue.java:194)
at java.base/java.util.concurrent.SynchronousQueue.xfer(SynchronousQueue.java:235)
at java.base/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:338)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1069)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
15) Thread[id=637, name=h2sc-262-thread-9, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
16) Thread[id=648, name=zkCallback-264-thread-1, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:410)
at java.base/java.util.concurrent.LinkedTransferQueue$DualNode.await(LinkedTransferQueue.java:452)
at java.base/java.util.concurrent.SynchronousQueue$Transferer.xferLifo(SynchronousQueue.java:194)
at java.base/java.util.concurrent.SynchronousQueue.xfer(SynchronousQueue.java:235)
at java.base/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:338)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1069)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
17) Thread[id=500, name=executor-Rebalance, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
18) Thread[id=48, name=NIOWorkerThread-3, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
19) Thread[id=525, name=h2sc-262-thread-6, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
20) Thread[id=49, name=SensorExpiryThread, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1763)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
21) Thread[id=63, name=data-plane-kafka-network-thread-0-ListenerName(PLAINTEXT)-PLAINTEXT-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at org.apache.kafka.common.metrics.Sensor.record(Sensor.java:226)
at org.apache.kafka.common.network.Selector.poll(Selector.java:512)
at kafka.network.Processor.poll(SocketServer.scala:1110)
at kafka.network.Processor.run(SocketServer.scala:1014)
at java.base/java.lang.Thread.run(Thread.java:1575)
22) Thread[id=62, name=data-plane-kafka-socket-acceptor-ListenerName(PLAINTEXT)-PLAINTEXT-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/java.lang.Thread.holdsLock(Native Method)
at java.base/sun.nio.ch.SelectorImpl.processDeregisterQueue(SelectorImpl.java:263)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:140)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at kafka.network.Acceptor.acceptNewConnections(SocketServer.scala:745)
at kafka.network.Acceptor.run(SocketServer.scala:693)
at java.base/java.lang.Thread.run(Thread.java:1575)
23) Thread[id=61, name=zk-broker-0-to-controller-forwarding-channel-manager, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/java.lang.Thread.holdsLock(Native Method)
at java.base/sun.nio.ch.SelectorImpl.processDeregisterQueue(SelectorImpl.java:263)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:140)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:596)
at org.apache.kafka.server.util.InterBrokerSendThread.pollOnce(InterBrokerSendThread.java:110)
at kafka.server.NodeToControllerRequestThread.doWork(NodeToControllerChannelManager.scala:377)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
24) Thread[id=44, name=SUITE-SolrAndKafkaReindexTest-seed#[A1D339859A09B250]-worker-SendThread(127.0.0.1:42491), state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/java.lang.Thread.sleepNanos0(Native Method)
at java.base/java.lang.Thread.sleepNanos(Thread.java:496)
at java.base/java.lang.Thread.sleep(Thread.java:527)
at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1137)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1200)
25) Thread[id=84, name=transaction-log-manager-0, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1763)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
26) Thread[id=472, name=CrossDCConsumer-255-thread-1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.unpark(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.unpark(LockSupport.java:181)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer.signalNext(AbstractQueuedSynchronizer.java:645)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer.release(AbstractQueuedSynchronizer.java:1060)
at java.base/java.util.concurrent.locks.ReentrantLock.unlock(ReentrantLock.java:495)
at org.gradle.internal.remote.internal.hub.MessageHub$ChannelDispatch.dispatch(MessageHub.java:375)
at org.gradle.internal.dispatch.ProxyDispatchAdapter$DispatchingInvocationHandler.invoke(ProxyDispatchAdapter.java:92)
at jdk.proxy1/jdk.proxy1.$Proxy5.output(Unknown Source)
at org.gradle.api.internal.tasks.testing.redirector.TestOutputRedirector$Forwarder.onOutput(TestOutputRedirector.java:74)
at org.gradle.api.internal.tasks.testing.redirector.DefaultStandardOutputRedirector$WriteAction.text(DefaultStandardOutputRedirector.java:85)
at org.gradle.internal.io.LineBufferingOutputStream.flush(LineBufferingOutputStream.java:97)
at org.gradle.internal.io.LineBufferingOutputStream.write(LineBufferingOutputStream.java:81)
at java.base/java.io.OutputStream.write(OutputStream.java:167)
at java.base/java.io.PrintStream.implWrite(PrintStream.java:647)
at java.base/java.io.PrintStream.write(PrintStream.java:627)
at org.gradle.internal.io.LinePerThreadBufferingOutputStream.write(LinePerThreadBufferingOutputStream.java:250)
at org.apache.lucene.tests.util.TestRuleLimitSysouts$DelegateStream.write(TestRuleLimitSysouts.java:173)
at java.base/java.io.PrintStream.implWrite(PrintStream.java:647)
at java.base/java.io.PrintStream.write(PrintStream.java:627)
at org.apache.logging.log4j.core.util.CloseShieldOutputStream.write(CloseShieldOutputStream.java:53)
at org.apache.logging.log4j.core.appender.OutputStreamManager.writeToDestination(OutputStreamManager.java:263)
at org.apache.logging.log4j.core.appender.OutputStreamManager.flushBuffer(OutputStreamManager.java:296)
at org.apache.logging.log4j.core.appender.OutputStreamManager.flush(OutputStreamManager.java:307)
at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:229)
at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:220)
at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:211)
at org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:160)
at org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:133)
at org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:124)
at org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:88)
at org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:711)
at org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:669)
at org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:645)
at org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:589)
at org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:92)
at org.apache.logging.log4j.core.Logger.log(Logger.java:187)
at org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2970)
at org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2922)
at org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2904)
at org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2648)
at org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:2587)
at org.apache.logging.slf4j.Log4jLogger.error(Log4jLogger.java:313)
at org.apache.solr.crossdc.manager.consumer.KafkaCrossDcConsumer.pollAndProcessRequests(KafkaCrossDcConsumer.java:563)
at org.apache.solr.crossdc.manager.consumer.KafkaCrossDcConsumer.run(KafkaCrossDcConsumer.java:291)
at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:408)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
27) Thread[id=34, name=NIOServerCxnFactory.AcceptThread:/127.0.0.1:0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.apache.zookeeper.server.NIOServerCnxnFactory$AcceptThread.select(NIOServerCnxnFactory.java:205)
at org.apache.zookeeper.server.NIOServerCnxnFactory$AcceptThread.run(NIOServerCnxnFactory.java:181)
28) Thread[id=47, name=NIOWorkerThread-2, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
29) Thread[id=66, name=zk-broker-0-to-controller-alter-partition-channel-manager, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EventFD.set0(Native Method)
at java.base/sun.nio.ch.EventFD.set(EventFD.java:48)
at java.base/sun.nio.ch.EPollSelectorImpl.wakeup(EPollSelectorImpl.java:257)
at java.base/java.nio.channels.spi.AbstractSelector$1.postInterrupt(AbstractSelector.java:115)
at java.base/java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:241)
at java.base/sun.nio.ch.SelectorImpl.begin(SelectorImpl.java:100)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:115)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:596)
at org.apache.kafka.server.util.InterBrokerSendThread.pollOnce(InterBrokerSendThread.java:110)
at kafka.server.NodeToControllerRequestThread.doWork(NodeToControllerChannelManager.scala:377)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
30) Thread[id=514, name=h2sc-262-thread-2-processing-collection1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:182)
at org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:189)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:616)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:550)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produceTask(AdaptiveExecutionStrategy.java:526)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:251)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produce(AdaptiveExecutionStrategy.java:196)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:408)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
31) Thread[id=563, name=executor-Fetch, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
32) Thread[id=523, name=h2sc-262-thread-4, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
33) Thread[id=83, name=TxnMarkerSenderThread-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:596)
at org.apache.kafka.server.util.InterBrokerSendThread.pollOnce(InterBrokerSendThread.java:110)
at org.apache.kafka.server.util.InterBrokerSendThread.doWork(InterBrokerSendThread.java:137)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
34) Thread[id=32, name=NIOServerCxnFactory.SelectorThread-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EventFD.set0(Native Method)
at java.base/sun.nio.ch.EventFD.set(EventFD.java:48)
at java.base/sun.nio.ch.EPollSelectorImpl.wakeup(EPollSelectorImpl.java:257)
at java.base/java.nio.channels.spi.AbstractSelector$1.postInterrupt(AbstractSelector.java:115)
at java.base/java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:241)
at java.base/sun.nio.ch.SelectorImpl.begin(SelectorImpl.java:100)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:115)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.select(NIOServerCnxnFactory.java:403)
at org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.run(NIOServerCnxnFactory.java:368)
35) Thread[id=513, name=h2sc-262-thread-1-processing-collection1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:182)
at org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:189)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:616)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:550)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produceTask(AdaptiveExecutionStrategy.java:526)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:251)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produce(AdaptiveExecutionStrategy.java:196)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:408)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
36) Thread[id=46, name=NIOWorkerThread-1, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
37) Thread[id=64, name=data-plane-kafka-network-thread-0-ListenerName(PLAINTEXT)-PLAINTEXT-1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/java.lang.Thread.holdsLock(Native Method)
at java.base/sun.nio.ch.SelectorImpl.processDeregisterQueue(SelectorImpl.java:263)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:140)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at kafka.network.Processor.poll(SocketServer.scala:1110)
at kafka.network.Processor.run(SocketServer.scala:1014)
at java.base/java.lang.Thread.run(Thread.java:1575)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=67, name=AddPartitionsToTxnSenderThread-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:596)
at org.apache.kafka.server.util.InterBrokerSendThread.pollOnce(InterBrokerSendThread.java:110)
at org.apache.kafka.server.util.InterBrokerSendThread.doWork(InterBrokerSendThread.java:137)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
2) Thread[id=82, name=group-metadata-manager-0, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1763)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
3) Thread[id=641, name=h2sc-262-thread-10, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
4) Thread[id=524, name=h2sc-262-thread-5, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
5) Thread[id=526, name=h2sc-262-thread-7, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
6) Thread[id=522, name=h2sc-262-thread-3, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
7) Thread[id=521, name=HttpClient@1f48b295-scheduler-1, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1763)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
8) Thread[id=527, name=h2sc-262-thread-8, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
9) Thread[id=518, name=Curator-Framework-0, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at java.base/java.util.concurrent.DelayQueue.take(DelayQueue.java:243)
at java.base/java.util.concurrent.DelayQueue.take(DelayQueue.java:100)
at org.apache.curator.framework.imps.CuratorFrameworkImpl.backgroundOperationsLoop(CuratorFrameworkImpl.java:796)
at org.apache.curator.framework.imps.CuratorFrameworkImpl.access$300(CuratorFrameworkImpl.java:77)
at org.apache.curator.framework.imps.CuratorFrameworkImpl$4.call(CuratorFrameworkImpl.java:320)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
10) Thread[id=33, name=NIOServerCxnFactory.SelectorThread-1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.IOUtil.drain(Native Method)
at java.base/sun.nio.ch.EventFD.reset(EventFD.java:52)
at java.base/sun.nio.ch.EPollSelectorImpl.clearInterrupt(EPollSelectorImpl.java:269)
at java.base/sun.nio.ch.EPollSelectorImpl.processEvents(EPollSelectorImpl.java:207)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:141)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.select(NIOServerCnxnFactory.java:403)
at org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.run(NIOServerCnxnFactory.java:368)
11) Thread[id=65, name=data-plane-kafka-network-thread-0-ListenerName(PLAINTEXT)-PLAINTEXT-2, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EventFD.set0(Native Method)
at java.base/sun.nio.ch.EventFD.set(EventFD.java:48)
at java.base/sun.nio.ch.EPollSelectorImpl.wakeup(EPollSelectorImpl.java:257)
at java.base/java.nio.channels.spi.AbstractSelector$1.postInterrupt(AbstractSelector.java:115)
at java.base/java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:241)
at java.base/sun.nio.ch.SelectorImpl.begin(SelectorImpl.java:100)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:115)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at kafka.network.Processor.poll(SocketServer.scala:1110)
at kafka.network.Processor.run(SocketServer.scala:1014)
at java.base/java.lang.Thread.run(Thread.java:1575)
12) Thread[id=693, name=zkCallback-264-thread-3, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:410)
at java.base/java.util.concurrent.LinkedTransferQueue$DualNode.await(LinkedTransferQueue.java:452)
at java.base/java.util.concurrent.SynchronousQueue$Transferer.xferLifo(SynchronousQueue.java:194)
at java.base/java.util.concurrent.SynchronousQueue.xfer(SynchronousQueue.java:235)
at java.base/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:338)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1069)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
13) Thread[id=515, name=Curator-ConnectionStateManager-0, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1763)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:435)
at org.apache.curator.framework.state.ConnectionStateManager.processEvents(ConnectionStateManager.java:246)
at org.apache.curator.framework.state.ConnectionStateManager.lambda$start$0(ConnectionStateManager.java:127)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
14) Thread[id=692, name=zkCallback-264-thread-2, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:410)
at java.base/java.util.concurrent.LinkedTransferQueue$DualNode.await(LinkedTransferQueue.java:452)
at java.base/java.util.concurrent.SynchronousQueue$Transferer.xferLifo(SynchronousQueue.java:194)
at java.base/java.util.concurrent.SynchronousQueue.xfer(SynchronousQueue.java:235)
at java.base/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:338)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1069)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
15) Thread[id=637, name=h2sc-262-thread-9, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
16) Thread[id=648, name=zkCallback-264-thread-1, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:410)
at java.base/java.util.concurrent.LinkedTransferQueue$DualNode.await(LinkedTransferQueue.java:452)
at java.base/java.util.concurrent.SynchronousQueue$Transferer.xferLifo(SynchronousQueue.java:194)
at java.base/java.util.concurrent.SynchronousQueue.xfer(SynchronousQueue.java:235)
at java.base/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:338)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1069)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
17) Thread[id=500, name=executor-Rebalance, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
18) Thread[id=48, name=NIOWorkerThread-3, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
19) Thread[id=525, name=h2sc-262-thread-6, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
20) Thread[id=49, name=SensorExpiryThread, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1763)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
21) Thread[id=63, name=data-plane-kafka-network-thread-0-ListenerName(PLAINTEXT)-PLAINTEXT-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at org.apache.kafka.common.metrics.Sensor.record(Sensor.java:226)
at org.apache.kafka.common.network.Selector.poll(Selector.java:512)
at kafka.network.Processor.poll(SocketServer.scala:1110)
at kafka.network.Processor.run(SocketServer.scala:1014)
at java.base/java.lang.Thread.run(Thread.java:1575)
22) Thread[id=62, name=data-plane-kafka-socket-acceptor-ListenerName(PLAINTEXT)-PLAINTEXT-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/java.lang.Thread.holdsLock(Native Method)
at java.base/sun.nio.ch.SelectorImpl.processDeregisterQueue(SelectorImpl.java:263)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:140)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at kafka.network.Acceptor.acceptNewConnections(SocketServer.scala:745)
at kafka.network.Acceptor.run(SocketServer.scala:693)
at java.base/java.lang.Thread.run(Thread.java:1575)
23) Thread[id=61, name=zk-broker-0-to-controller-forwarding-channel-manager, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/java.lang.Thread.holdsLock(Native Method)
at java.base/sun.nio.ch.SelectorImpl.processDeregisterQueue(SelectorImpl.java:263)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:140)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:596)
at org.apache.kafka.server.util.InterBrokerSendThread.pollOnce(InterBrokerSendThread.java:110)
at kafka.server.NodeToControllerRequestThread.doWork(NodeToControllerChannelManager.scala:377)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
24) Thread[id=44, name=SUITE-SolrAndKafkaReindexTest-seed#[A1D339859A09B250]-worker-SendThread(127.0.0.1:42491), state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/java.lang.Thread.sleepNanos0(Native Method)
at java.base/java.lang.Thread.sleepNanos(Thread.java:496)
at java.base/java.lang.Thread.sleep(Thread.java:527)
at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1137)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1200)
25) Thread[id=84, name=transaction-log-manager-0, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1763)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
26) Thread[id=472, name=CrossDCConsumer-255-thread-1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.unpark(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.unpark(LockSupport.java:181)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer.signalNext(AbstractQueuedSynchronizer.java:645)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer.release(AbstractQueuedSynchronizer.java:1060)
at java.base/java.util.concurrent.locks.ReentrantLock.unlock(ReentrantLock.java:495)
at org.gradle.internal.remote.internal.hub.MessageHub$ChannelDispatch.dispatch(MessageHub.java:375)
at org.gradle.internal.dispatch.ProxyDispatchAdapter$DispatchingInvocationHandler.invoke(ProxyDispatchAdapter.java:92)
at jdk.proxy1/jdk.proxy1.$Proxy5.output(Unknown Source)
at org.gradle.api.internal.tasks.testing.redirector.TestOutputRedirector$Forwarder.onOutput(TestOutputRedirector.java:74)
at org.gradle.api.internal.tasks.testing.redirector.DefaultStandardOutputRedirector$WriteAction.text(DefaultStandardOutputRedirector.java:85)
at org.gradle.internal.io.LineBufferingOutputStream.flush(LineBufferingOutputStream.java:97)
at org.gradle.internal.io.LineBufferingOutputStream.write(LineBufferingOutputStream.java:81)
at java.base/java.io.OutputStream.write(OutputStream.java:167)
at java.base/java.io.PrintStream.implWrite(PrintStream.java:647)
at java.base/java.io.PrintStream.write(PrintStream.java:627)
at org.gradle.internal.io.LinePerThreadBufferingOutputStream.write(LinePerThreadBufferingOutputStream.java:250)
at org.apache.lucene.tests.util.TestRuleLimitSysouts$DelegateStream.write(TestRuleLimitSysouts.java:173)
at java.base/java.io.PrintStream.implWrite(PrintStream.java:647)
at java.base/java.io.PrintStream.write(PrintStream.java:627)
at org.apache.logging.log4j.core.util.CloseShieldOutputStream.write(CloseShieldOutputStream.java:53)
at org.apache.logging.log4j.core.appender.OutputStreamManager.writeToDestination(OutputStreamManager.java:263)
at org.apache.logging.log4j.core.appender.OutputStreamManager.flushBuffer(OutputStreamManager.java:296)
at org.apache.logging.log4j.core.appender.OutputStreamManager.flush(OutputStreamManager.java:307)
at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.directEncodeEvent(AbstractOutputStreamAppender.java:229)
at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.tryAppend(AbstractOutputStreamAppender.java:220)
at org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender.append(AbstractOutputStreamAppender.java:211)
at org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:160)
at org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:133)
at org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:124)
at org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:88)
at org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:711)
at org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:669)
at org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:645)
at org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:589)
at org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:92)
at org.apache.logging.log4j.core.Logger.log(Logger.java:187)
at org.apache.logging.log4j.spi.AbstractLogger.tryLogMessage(AbstractLogger.java:2970)
at org.apache.logging.log4j.spi.AbstractLogger.logMessageTrackRecursion(AbstractLogger.java:2922)
at org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2904)
at org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:2648)
at org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:2587)
at org.apache.logging.slf4j.Log4jLogger.error(Log4jLogger.java:313)
at org.apache.solr.crossdc.manager.consumer.KafkaCrossDcConsumer.pollAndProcessRequests(KafkaCrossDcConsumer.java:563)
at org.apache.solr.crossdc.manager.consumer.KafkaCrossDcConsumer.run(KafkaCrossDcConsumer.java:291)
at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:408)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
27) Thread[id=34, name=NIOServerCxnFactory.AcceptThread:/127.0.0.1:0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.apache.zookeeper.server.NIOServerCnxnFactory$AcceptThread.select(NIOServerCnxnFactory.java:205)
at org.apache.zookeeper.server.NIOServerCnxnFactory$AcceptThread.run(NIOServerCnxnFactory.java:181)
28) Thread[id=47, name=NIOWorkerThread-2, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
29) Thread[id=66, name=zk-broker-0-to-controller-alter-partition-channel-manager, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EventFD.set0(Native Method)
at java.base/sun.nio.ch.EventFD.set(EventFD.java:48)
at java.base/sun.nio.ch.EPollSelectorImpl.wakeup(EPollSelectorImpl.java:257)
at java.base/java.nio.channels.spi.AbstractSelector$1.postInterrupt(AbstractSelector.java:115)
at java.base/java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:241)
at java.base/sun.nio.ch.SelectorImpl.begin(SelectorImpl.java:100)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:115)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:596)
at org.apache.kafka.server.util.InterBrokerSendThread.pollOnce(InterBrokerSendThread.java:110)
at kafka.server.NodeToControllerRequestThread.doWork(NodeToControllerChannelManager.scala:377)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
30) Thread[id=514, name=h2sc-262-thread-2-processing-collection1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:182)
at org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:189)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:616)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:550)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produceTask(AdaptiveExecutionStrategy.java:526)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:251)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produce(AdaptiveExecutionStrategy.java:196)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:408)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
31) Thread[id=563, name=executor-Fetch, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
32) Thread[id=523, name=h2sc-262-thread-4, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
33) Thread[id=83, name=TxnMarkerSenderThread-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:596)
at org.apache.kafka.server.util.InterBrokerSendThread.pollOnce(InterBrokerSendThread.java:110)
at org.apache.kafka.server.util.InterBrokerSendThread.doWork(InterBrokerSendThread.java:137)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
34) Thread[id=32, name=NIOServerCxnFactory.SelectorThread-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EventFD.set0(Native Method)
at java.base/sun.nio.ch.EventFD.set(EventFD.java:48)
at java.base/sun.nio.ch.EPollSelectorImpl.wakeup(EPollSelectorImpl.java:257)
at java.base/java.nio.channels.spi.AbstractSelector$1.postInterrupt(AbstractSelector.java:115)
at java.base/java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:241)
at java.base/sun.nio.ch.SelectorImpl.begin(SelectorImpl.java:100)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:115)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.select(NIOServerCnxnFactory.java:403)
at org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.run(NIOServerCnxnFactory.java:368)
35) Thread[id=513, name=h2sc-262-thread-1-processing-collection1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:182)
at org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:189)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:616)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:550)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produceTask(AdaptiveExecutionStrategy.java:526)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:251)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produce(AdaptiveExecutionStrategy.java:196)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:408)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
36) Thread[id=46, name=NIOWorkerThread-1, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
37) Thread[id=64, name=data-plane-kafka-network-thread-0-ListenerName(PLAINTEXT)-PLAINTEXT-1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/java.lang.Thread.holdsLock(Native Method)
at java.base/sun.nio.ch.SelectorImpl.processDeregisterQueue(SelectorImpl.java:263)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:140)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at kafka.network.Processor.poll(SocketServer.scala:1110)
at kafka.network.Processor.run(SocketServer.scala:1014)
at java.base/java.lang.Thread.run(Thread.java:1575)
at __randomizedtesting.SeedInfo.seed([A1D339859A09B250]:0)
FAILED: org.apache.solr.crossdc.manager.SolrAndKafkaReindexTest.classMethod
Error Message:
com.carrotsearch.randomizedtesting.ThreadLeakError: 67 threads leaked from SUITE scope at org.apache.solr.crossdc.manager.SolrAndKafkaReindexTest:
1) Thread[id=67, name=AddPartitionsToTxnSenderThread-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:596)
at org.apache.kafka.server.util.InterBrokerSendThread.pollOnce(InterBrokerSendThread.java:110)
at org.apache.kafka.server.util.InterBrokerSendThread.doWork(InterBrokerSendThread.java:137)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
2) Thread[id=82, name=group-metadata-manager-0, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1763)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
3) Thread[id=641, name=h2sc-262-thread-10, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
4) Thread[id=38, name=RequestThrottler, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at org.apache.zookeeper.server.RequestThrottler.run(RequestThrottler.java:147)
5) Thread[id=71, name=ExpirationReaper-0-ElectLeader, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1763)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:292)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
6) Thread[id=526, name=h2sc-262-thread-7, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
7) Thread[id=93, name=data-plane-kafka-request-handler-6, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1763)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:435)
at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:484)
at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:113)
at java.base/java.lang.Thread.run(Thread.java:1575)
8) Thread[id=90, name=data-plane-kafka-request-handler-3, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1763)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:435)
at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:484)
at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:113)
at java.base/java.lang.Thread.run(Thread.java:1575)
9) Thread[id=515, name=Curator-ConnectionStateManager-0, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1763)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:435)
at org.apache.curator.framework.state.ConnectionStateManager.processEvents(ConnectionStateManager.java:246)
at org.apache.curator.framework.state.ConnectionStateManager.lambda$start$0(ConnectionStateManager.java:127)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
10) Thread[id=76, name=LogDirFailureHandler, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at java.base/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:420)
at org.apache.kafka.storage.internals.log.LogDirFailureChannel.takeNextOfflineLogDir(LogDirFailureChannel.java:75)
at kafka.server.ReplicaManager$LogDirFailureHandler.doWork(ReplicaManager.scala:320)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
11) Thread[id=648, name=zkCallback-264-thread-1, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:410)
at java.base/java.util.concurrent.LinkedTransferQueue$DualNode.await(LinkedTransferQueue.java:452)
at java.base/java.util.concurrent.SynchronousQueue$Transferer.xferLifo(SynchronousQueue.java:194)
at java.base/java.util.concurrent.SynchronousQueue.xfer(SynchronousQueue.java:235)
at java.base/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:338)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1069)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
12) Thread[id=95, name=/config/changes-event-process-thread, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at kafka.common.ZkNodeChangeNotificationListener$ChangeEventProcessThread.doWork(ZkNodeChangeNotificationListener.scala:147)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
13) Thread[id=500, name=executor-Rebalance, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
14) Thread[id=63, name=data-plane-kafka-network-thread-0-ListenerName(PLAINTEXT)-PLAINTEXT-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at kafka.network.Processor.poll(SocketServer.scala:1110)
at kafka.network.Processor.run(SocketServer.scala:1014)
at java.base/java.lang.Thread.run(Thread.java:1575)
15) Thread[id=70, name=ExpirationReaper-0-DeleteRecords, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1763)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:292)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
16) Thread[id=61, name=zk-broker-0-to-controller-forwarding-channel-manager, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:596)
at org.apache.kafka.server.util.InterBrokerSendThread.pollOnce(InterBrokerSendThread.java:110)
at kafka.server.NodeToControllerRequestThread.doWork(NodeToControllerChannelManager.scala:377)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
17) Thread[id=44, name=SUITE-SolrAndKafkaReindexTest-seed#[A1D339859A09B250]-worker-SendThread(127.0.0.1:42491), state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1290)
18) Thread[id=84, name=transaction-log-manager-0, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1763)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
19) Thread[id=472, name=CrossDCConsumer-255-thread-1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:596)
at org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.poll(ConsumerNetworkClient.java:282)
at org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.poll(ConsumerNetworkClient.java:253)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:692)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.solr.crossdc.manager.consumer.KafkaCrossDcConsumer.pollAndProcessRequests(KafkaCrossDcConsumer.java:372)
at org.apache.solr.crossdc.manager.consumer.KafkaCrossDcConsumer.run(KafkaCrossDcConsumer.java:291)
at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:408)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
20) Thread[id=47, name=NIOWorkerThread-2, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
21) Thread[id=69, name=ExpirationReaper-0-Fetch, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1763)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:301)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
22) Thread[id=51, name=ThrottledChannelReaper-Produce, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1763)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:292)
at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:178)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
23) Thread[id=66, name=zk-broker-0-to-controller-alter-partition-channel-manager, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:596)
at org.apache.kafka.server.util.InterBrokerSendThread.pollOnce(InterBrokerSendThread.java:110)
at kafka.server.NodeToControllerRequestThread.doWork(NodeToControllerChannelManager.scala:377)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
24) Thread[id=86, name=ExpirationReaper-0-AlterAcls, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1763)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:292)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
25) Thread[id=53, name=ThrottledChannelReaper-ControllerMutation, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1763)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:292)
at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:178)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
26) Thread[id=83, name=TxnMarkerSenderThread-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:596)
at org.apache.kafka.server.util.InterBrokerSendThread.pollOnce(InterBrokerSendThread.java:110)
at org.apache.kafka.server.util.InterBrokerSendThread.doWork(InterBrokerSendThread.java:137)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
27) Thread[id=32, name=NIOServerCxnFactory.SelectorThread-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.select(NIOServerCnxnFactory.java:403)
at org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.run(NIOServerCnxnFactory.java:368)
28) Thread[id=64, name=data-plane-kafka-network-thread-0-ListenerName(PLAINTEXT)-PLAINTEXT-1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at kafka.network.Processor.poll(SocketServer.scala:1110)
at kafka.network.Processor.run(SocketServer.scala:1014)
at java.base/java.lang.Thread.run(Thread.java:1575)
29) Thread[id=524, name=h2sc-262-thread-5, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
30) Thread[id=45, name=SUITE-SolrAndKafkaReindexTest-seed#[A1D339859A09B250]-worker-EventThread, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:537)
31) Thread[id=92, name=data-plane-kafka-request-handler-5, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1763)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:435)
at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:484)
at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:113)
at java.base/java.lang.Thread.run(Thread.java:1575)
32) Thread[id=522, name=h2sc-262-thread-3, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
33) Thread[id=60, name=feature-zk-node-event-process-thread, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at kafka.server.FinalizedFeatureChangeListener$ChangeNotificationProcessorThread.doWork(FinalizedFeatureChangeListener.scala:154)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
34) Thread[id=87, name=data-plane-kafka-request-handler-0, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1763)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:435)
at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:484)
at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:113)
at java.base/java.lang.Thread.run(Thread.java:1575)
35) Thread[id=521, name=HttpClient@1f48b295-scheduler-1, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1763)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
36) Thread[id=527, name=h2sc-262-thread-8, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
37) Thread[id=518, name=Curator-Framework-0, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at java.base/java.util.concurrent.DelayQueue.take(DelayQueue.java:243)
at java.base/java.util.concurrent.DelayQueue.take(DelayQueue.java:100)
at org.apache.curator.framework.imps.CuratorFrameworkImpl.backgroundOperationsLoop(CuratorFrameworkImpl.java:796)
at org.apache.curator.framework.imps.CuratorFrameworkImpl.access$300(CuratorFrameworkImpl.java:77)
at org.apache.curator.framework.imps.CuratorFrameworkImpl$4.call(CuratorFrameworkImpl.java:320)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
38) Thread[id=33, name=NIOServerCxnFactory.SelectorThread-1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.select(NIOServerCnxnFactory.java:403)
at org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.run(NIOServerCnxnFactory.java:368)
39) Thread[id=65, name=data-plane-kafka-network-thread-0-ListenerName(PLAINTEXT)-PLAINTEXT-2, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at kafka.network.Processor.poll(SocketServer.scala:1110)
at kafka.network.Processor.run(SocketServer.scala:1014)
at java.base/java.lang.Thread.run(Thread.java:1575)
40) Thread[id=693, name=zkCallback-264-thread-3, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:410)
at java.base/java.util.concurrent.LinkedTransferQueue$DualNode.await(LinkedTransferQueue.java:452)
at java.base/java.util.concurrent.SynchronousQueue$Transferer.xferLifo(SynchronousQueue.java:194)
at java.base/java.util.concurrent.SynchronousQueue.xfer(SynchronousQueue.java:235)
at java.base/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:338)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1069)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
41) Thread[id=50, name=ThrottledChannelReaper-Fetch, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1763)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:292)
at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:178)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
42) Thread[id=80, name=ExpirationReaper-0-Heartbeat, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1763)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:301)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
43) Thread[id=692, name=zkCallback-264-thread-2, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:410)
at java.base/java.util.concurrent.LinkedTransferQueue$DualNode.await(LinkedTransferQueue.java:452)
at java.base/java.util.concurrent.SynchronousQueue$Transferer.xferLifo(SynchronousQueue.java:194)
at java.base/java.util.concurrent.SynchronousQueue.xfer(SynchronousQueue.java:235)
at java.base/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:338)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1069)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
44) Thread[id=637, name=h2sc-262-thread-9, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
45) Thread[id=78, name=controller-event-thread, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1763)
at java.base/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
at kafka.controller.ControllerEventManager.kafka$controller$ControllerEventManager$$pollFromEventQueue(ControllerEventManager.scala:156)
at kafka.controller.ControllerEventManager$ControllerEventThread.doWork(ControllerEventManager.scala:129)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
46) Thread[id=48, name=NIOWorkerThread-3, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
47) Thread[id=525, name=h2sc-262-thread-6, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
48) Thread[id=49, name=SensorExpiryThread, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1763)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
49) Thread[id=72, name=ExpirationReaper-0-RemoteFetch, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1763)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:292)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
50) Thread[id=94, name=data-plane-kafka-request-handler-7, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1763)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:435)
at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:484)
at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:113)
at java.base/java.lang.Thread.run(Thread.java:1575)
51) Thread[id=81, name=ExpirationReaper-0-Rebalance, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1763)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:292)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
52) Thread[id=62, name=data-plane-kafka-socket-acceptor-ListenerName(PLAINTEXT)-PLAINTEXT-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at kafka.network.Acceptor.acceptNewConnections(SocketServer.scala:745)
at kafka.network.Acceptor.run(SocketServer.scala:693)
at java.base/java.lang.Thread.run(Thread.java:1575)
53) Thread[id=34, name=NIOServerCxnFactory.AcceptThread:/127.0.0.1:0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.apache.zookeeper.server.NIOServerCnxnFactory$AcceptThread.select(NIOServerCnxnFactory.java:205)
at org.apache.zookeeper.server.NIOServerCnxnFactory$AcceptThread.run(NIOServerCnxnFactory.java:181)
54) Thread[id=37, name=ProcessThread(sid:0 cport:42491):, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:142)
55) Thread[id=89, name=data-plane-kafka-request-handler-2, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1763)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:435)
at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:484)
at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:113)
at java.base/java.lang.Thread.run(Thread.java:1575)
56) Thread[id=514, name=h2sc-262-thread-2-processing-collection1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:182)
at org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:189)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:616)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:550)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produceTask(AdaptiveExecutionStrategy.java:526)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:251)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produce(AdaptiveExecutionStrategy.java:196)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:408)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
57) Thread[id=563, name=executor-Fetch, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
58) Thread[id=523, name=h2sc-262-thread-4, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
59) Thread[id=68, name=ExpirationReaper-0-Produce, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1763)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:292)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
60) Thread[id=91, name=data-plane-kafka-request-handler-4, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1763)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:435)
at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:484)
at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:113)
at java.base/java.lang.Thread.run(Thread.java:1575)
61) Thread[id=36, name=SyncThread:0, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at org.apache.zookeeper.server.SyncRequestProcessor.run(SyncRequestProcessor.java:170)
62) Thread[id=79, name=ExpirationReaper-0-topic, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1763)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:292)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
63) Thread[id=513, name=h2sc-262-thread-1-processing-collection1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:182)
at org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:189)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:616)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:550)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produceTask(AdaptiveExecutionStrategy.java:526)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:251)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produce(AdaptiveExecutionStrategy.java:196)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:408)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
64) Thread[id=52, name=ThrottledChannelReaper-Request, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1763)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:292)
at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:178)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
65) Thread[id=85, name=Controller-0-to-broker-0-send-thread, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:242)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
66) Thread[id=88, name=data-plane-kafka-request-handler-1, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1763)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:435)
at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:484)
at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:113)
at java.base/java.lang.Thread.run(Thread.java:1575)
67) Thread[id=46, name=NIOWorkerThread-1, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 67 threads leaked from SUITE scope at org.apache.solr.crossdc.manager.SolrAndKafkaReindexTest:
1) Thread[id=67, name=AddPartitionsToTxnSenderThread-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:596)
at org.apache.kafka.server.util.InterBrokerSendThread.pollOnce(InterBrokerSendThread.java:110)
at org.apache.kafka.server.util.InterBrokerSendThread.doWork(InterBrokerSendThread.java:137)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
2) Thread[id=82, name=group-metadata-manager-0, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1763)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
3) Thread[id=641, name=h2sc-262-thread-10, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
4) Thread[id=38, name=RequestThrottler, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at org.apache.zookeeper.server.RequestThrottler.run(RequestThrottler.java:147)
5) Thread[id=71, name=ExpirationReaper-0-ElectLeader, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1763)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:292)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
6) Thread[id=526, name=h2sc-262-thread-7, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
7) Thread[id=93, name=data-plane-kafka-request-handler-6, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1763)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:435)
at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:484)
at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:113)
at java.base/java.lang.Thread.run(Thread.java:1575)
8) Thread[id=90, name=data-plane-kafka-request-handler-3, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1763)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:435)
at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:484)
at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:113)
at java.base/java.lang.Thread.run(Thread.java:1575)
9) Thread[id=515, name=Curator-ConnectionStateManager-0, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1763)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:435)
at org.apache.curator.framework.state.ConnectionStateManager.processEvents(ConnectionStateManager.java:246)
at org.apache.curator.framework.state.ConnectionStateManager.lambda$start$0(ConnectionStateManager.java:127)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
10) Thread[id=76, name=LogDirFailureHandler, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at java.base/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:420)
at org.apache.kafka.storage.internals.log.LogDirFailureChannel.takeNextOfflineLogDir(LogDirFailureChannel.java:75)
at kafka.server.ReplicaManager$LogDirFailureHandler.doWork(ReplicaManager.scala:320)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
11) Thread[id=648, name=zkCallback-264-thread-1, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:410)
at java.base/java.util.concurrent.LinkedTransferQueue$DualNode.await(LinkedTransferQueue.java:452)
at java.base/java.util.concurrent.SynchronousQueue$Transferer.xferLifo(SynchronousQueue.java:194)
at java.base/java.util.concurrent.SynchronousQueue.xfer(SynchronousQueue.java:235)
at java.base/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:338)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1069)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
12) Thread[id=95, name=/config/changes-event-process-thread, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at kafka.common.ZkNodeChangeNotificationListener$ChangeEventProcessThread.doWork(ZkNodeChangeNotificationListener.scala:147)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
13) Thread[id=500, name=executor-Rebalance, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
14) Thread[id=63, name=data-plane-kafka-network-thread-0-ListenerName(PLAINTEXT)-PLAINTEXT-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at kafka.network.Processor.poll(SocketServer.scala:1110)
at kafka.network.Processor.run(SocketServer.scala:1014)
at java.base/java.lang.Thread.run(Thread.java:1575)
15) Thread[id=70, name=ExpirationReaper-0-DeleteRecords, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1763)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:292)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
16) Thread[id=61, name=zk-broker-0-to-controller-forwarding-channel-manager, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:596)
at org.apache.kafka.server.util.InterBrokerSendThread.pollOnce(InterBrokerSendThread.java:110)
at kafka.server.NodeToControllerRequestThread.doWork(NodeToControllerChannelManager.scala:377)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
17) Thread[id=44, name=SUITE-SolrAndKafkaReindexTest-seed#[A1D339859A09B250]-worker-SendThread(127.0.0.1:42491), state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1290)
18) Thread[id=84, name=transaction-log-manager-0, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1763)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
19) Thread[id=472, name=CrossDCConsumer-255-thread-1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:596)
at org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.poll(ConsumerNetworkClient.java:282)
at org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.poll(ConsumerNetworkClient.java:253)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.pollForFetches(ClassicKafkaConsumer.java:692)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:623)
at org.apache.kafka.clients.consumer.internals.ClassicKafkaConsumer.poll(ClassicKafkaConsumer.java:596)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:874)
at org.apache.solr.crossdc.manager.consumer.KafkaCrossDcConsumer.pollAndProcessRequests(KafkaCrossDcConsumer.java:372)
at org.apache.solr.crossdc.manager.consumer.KafkaCrossDcConsumer.run(KafkaCrossDcConsumer.java:291)
at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:408)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
20) Thread[id=47, name=NIOWorkerThread-2, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
21) Thread[id=69, name=ExpirationReaper-0-Fetch, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1763)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:301)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
22) Thread[id=51, name=ThrottledChannelReaper-Produce, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1763)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:292)
at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:178)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
23) Thread[id=66, name=zk-broker-0-to-controller-alter-partition-channel-manager, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:596)
at org.apache.kafka.server.util.InterBrokerSendThread.pollOnce(InterBrokerSendThread.java:110)
at kafka.server.NodeToControllerRequestThread.doWork(NodeToControllerChannelManager.scala:377)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
24) Thread[id=86, name=ExpirationReaper-0-AlterAcls, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1763)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:292)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
25) Thread[id=53, name=ThrottledChannelReaper-ControllerMutation, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1763)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:292)
at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:178)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
26) Thread[id=83, name=TxnMarkerSenderThread-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:596)
at org.apache.kafka.server.util.InterBrokerSendThread.pollOnce(InterBrokerSendThread.java:110)
at org.apache.kafka.server.util.InterBrokerSendThread.doWork(InterBrokerSendThread.java:137)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
27) Thread[id=32, name=NIOServerCxnFactory.SelectorThread-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.select(NIOServerCnxnFactory.java:403)
at org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.run(NIOServerCnxnFactory.java:368)
28) Thread[id=64, name=data-plane-kafka-network-thread-0-ListenerName(PLAINTEXT)-PLAINTEXT-1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at kafka.network.Processor.poll(SocketServer.scala:1110)
at kafka.network.Processor.run(SocketServer.scala:1014)
at java.base/java.lang.Thread.run(Thread.java:1575)
29) Thread[id=524, name=h2sc-262-thread-5, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
30) Thread[id=45, name=SUITE-SolrAndKafkaReindexTest-seed#[A1D339859A09B250]-worker-EventThread, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:537)
31) Thread[id=92, name=data-plane-kafka-request-handler-5, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1763)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:435)
at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:484)
at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:113)
at java.base/java.lang.Thread.run(Thread.java:1575)
32) Thread[id=522, name=h2sc-262-thread-3, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
33) Thread[id=60, name=feature-zk-node-event-process-thread, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at kafka.server.FinalizedFeatureChangeListener$ChangeNotificationProcessorThread.doWork(FinalizedFeatureChangeListener.scala:154)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
34) Thread[id=87, name=data-plane-kafka-request-handler-0, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1763)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:435)
at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:484)
at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:113)
at java.base/java.lang.Thread.run(Thread.java:1575)
35) Thread[id=521, name=HttpClient@1f48b295-scheduler-1, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1763)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
36) Thread[id=527, name=h2sc-262-thread-8, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
37) Thread[id=518, name=Curator-Framework-0, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at java.base/java.util.concurrent.DelayQueue.take(DelayQueue.java:243)
at java.base/java.util.concurrent.DelayQueue.take(DelayQueue.java:100)
at org.apache.curator.framework.imps.CuratorFrameworkImpl.backgroundOperationsLoop(CuratorFrameworkImpl.java:796)
at org.apache.curator.framework.imps.CuratorFrameworkImpl.access$300(CuratorFrameworkImpl.java:77)
at org.apache.curator.framework.imps.CuratorFrameworkImpl$4.call(CuratorFrameworkImpl.java:320)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
38) Thread[id=33, name=NIOServerCxnFactory.SelectorThread-1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.select(NIOServerCnxnFactory.java:403)
at org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.run(NIOServerCnxnFactory.java:368)
39) Thread[id=65, name=data-plane-kafka-network-thread-0-ListenerName(PLAINTEXT)-PLAINTEXT-2, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at org.apache.kafka.common.network.Selector.select(Selector.java:895)
at org.apache.kafka.common.network.Selector.poll(Selector.java:485)
at kafka.network.Processor.poll(SocketServer.scala:1110)
at kafka.network.Processor.run(SocketServer.scala:1014)
at java.base/java.lang.Thread.run(Thread.java:1575)
40) Thread[id=693, name=zkCallback-264-thread-3, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:410)
at java.base/java.util.concurrent.LinkedTransferQueue$DualNode.await(LinkedTransferQueue.java:452)
at java.base/java.util.concurrent.SynchronousQueue$Transferer.xferLifo(SynchronousQueue.java:194)
at java.base/java.util.concurrent.SynchronousQueue.xfer(SynchronousQueue.java:235)
at java.base/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:338)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1069)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
41) Thread[id=50, name=ThrottledChannelReaper-Fetch, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1763)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:292)
at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:178)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
42) Thread[id=80, name=ExpirationReaper-0-Heartbeat, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1763)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:301)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
43) Thread[id=692, name=zkCallback-264-thread-2, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:410)
at java.base/java.util.concurrent.LinkedTransferQueue$DualNode.await(LinkedTransferQueue.java:452)
at java.base/java.util.concurrent.SynchronousQueue$Transferer.xferLifo(SynchronousQueue.java:194)
at java.base/java.util.concurrent.SynchronousQueue.xfer(SynchronousQueue.java:235)
at java.base/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:338)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1069)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
44) Thread[id=637, name=h2sc-262-thread-9, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
45) Thread[id=78, name=controller-event-thread, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1763)
at java.base/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
at kafka.controller.ControllerEventManager.kafka$controller$ControllerEventManager$$pollFromEventQueue(ControllerEventManager.scala:156)
at kafka.controller.ControllerEventManager$ControllerEventThread.doWork(ControllerEventManager.scala:129)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
46) Thread[id=48, name=NIOWorkerThread-3, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
47) Thread[id=525, name=h2sc-262-thread-6, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
48) Thread[id=49, name=SensorExpiryThread, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1763)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
49) Thread[id=72, name=ExpirationReaper-0-RemoteFetch, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1763)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:292)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
50) Thread[id=94, name=data-plane-kafka-request-handler-7, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1763)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:435)
at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:484)
at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:113)
at java.base/java.lang.Thread.run(Thread.java:1575)
51) Thread[id=81, name=ExpirationReaper-0-Rebalance, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1763)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:292)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
52) Thread[id=62, name=data-plane-kafka-socket-acceptor-ListenerName(PLAINTEXT)-PLAINTEXT-0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:142)
at kafka.network.Acceptor.acceptNewConnections(SocketServer.scala:745)
at kafka.network.Acceptor.run(SocketServer.scala:693)
at java.base/java.lang.Thread.run(Thread.java:1575)
53) Thread[id=34, name=NIOServerCxnFactory.AcceptThread:/127.0.0.1:0, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.apache.zookeeper.server.NIOServerCnxnFactory$AcceptThread.select(NIOServerCnxnFactory.java:205)
at org.apache.zookeeper.server.NIOServerCnxnFactory$AcceptThread.run(NIOServerCnxnFactory.java:181)
54) Thread[id=37, name=ProcessThread(sid:0 cport:42491):, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:142)
55) Thread[id=89, name=data-plane-kafka-request-handler-2, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1763)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:435)
at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:484)
at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:113)
at java.base/java.lang.Thread.run(Thread.java:1575)
56) Thread[id=514, name=h2sc-262-thread-2-processing-collection1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:182)
at org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:189)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:616)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:550)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produceTask(AdaptiveExecutionStrategy.java:526)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:251)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produce(AdaptiveExecutionStrategy.java:196)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:408)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
57) Thread[id=563, name=executor-Fetch, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
58) Thread[id=523, name=h2sc-262-thread-4, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at org.eclipse.jetty.util.BlockingArrayQueue.take(BlockingArrayQueue.java:534)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
59) Thread[id=68, name=ExpirationReaper-0-Produce, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1763)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:292)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
60) Thread[id=91, name=data-plane-kafka-request-handler-4, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1763)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:435)
at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:484)
at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:113)
at java.base/java.lang.Thread.run(Thread.java:1575)
61) Thread[id=36, name=SyncThread:0, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at org.apache.zookeeper.server.SyncRequestProcessor.run(SyncRequestProcessor.java:170)
62) Thread[id=79, name=ExpirationReaper-0-topic, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1763)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:292)
at org.apache.kafka.server.util.timer.SystemTimer.advanceClock(SystemTimer.java:90)
at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:418)
at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:444)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
63) Thread[id=513, name=h2sc-262-thread-1-processing-collection1, state=RUNNABLE, group=TGRP-SolrAndKafkaReindexTest]
at java.base/sun.nio.ch.EPoll.wait(Native Method)
at java.base/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:121)
at java.base/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:130)
at java.base/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:147)
at org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:182)
at org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:189)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:616)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:550)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produceTask(AdaptiveExecutionStrategy.java:526)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:251)
at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.produce(AdaptiveExecutionStrategy.java:196)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:408)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
64) Thread[id=52, name=ThrottledChannelReaper-Request, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1763)
at java.base/java.util.concurrent.DelayQueue.poll(DelayQueue.java:292)
at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:178)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
65) Thread[id=85, name=Controller-0-to-broker-0-send-thread, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:242)
at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:136)
66) Thread[id=88, name=data-plane-kafka-request-handler-1, state=TIMED_WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:269)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1763)
at java.base/java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:435)
at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:484)
at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:113)
at java.base/java.lang.Thread.run(Thread.java:1575)
67) Thread[id=46, name=NIOWorkerThread-1, state=WAITING, group=TGRP-SolrAndKafkaReindexTest]
at java.base/jdk.internal.misc.Unsafe.park(Native Method)
at java.base/java.util.concurrent.locks.LockSupport.park(LockSupport.java:371)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:519)
at java.base/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:4023)
at java.base/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3969)
at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1712)
at java.base/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1070)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
at __randomizedtesting.SeedInfo.seed([A1D339859A09B250]:0)
FAILED: org.apache.solr.crossdc.manager.SolrAndKafkaReindexTest.classMethod
Error Message:
java.lang.Exception: Error shutting down MiniSolrCloudCluster
Stack Trace:
java.lang.Exception: Error shutting down MiniSolrCloudCluster
at __randomizedtesting.SeedInfo.seed([A1D339859A09B250]:0)
at org.apache.solr.cloud.MiniSolrCloudCluster.checkForExceptions(MiniSolrCloudCluster.java:714)
at org.apache.solr.cloud.MiniSolrCloudCluster.shutdown(MiniSolrCloudCluster.java:635)
at org.apache.solr.crossdc.manager.SolrAndKafkaReindexTest.afterSolrAndKafkaIntegrationTest(SolrAndKafkaReindexTest.java:134)
at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103)
at java.base/java.lang.reflect.Method.invoke(Method.java:580)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1763)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:909)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
at org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
at java.base/java.lang.Thread.run(Thread.java:1575)
Suppressed: java.lang.RuntimeException: Timeout waiting for pool to shutdown. [poolName=closeThreadPool][poolName=closeThreadPool] org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor@5b417e3b[Shutting down, pool size = 1, active threads = 1, queued tasks = 0, completed tasks = 4]
at org.apache.solr.common.util.ExecutorUtil.awaitTermination(ExecutorUtil.java:167)
at org.apache.solr.common.util.ExecutorUtil.awaitTermination(ExecutorUtil.java:153)
at org.apache.solr.common.util.ExecutorUtil.shutdownAndAwaitTermination(ExecutorUtil.java:119)
at org.apache.solr.core.CoreContainer.shutdown(CoreContainer.java:1325)
at org.apache.solr.servlet.CoreContainerProvider.close(CoreContainerProvider.java:132)
at org.apache.solr.servlet.CoreContainerProvider.contextDestroyed(CoreContainerProvider.java:90)
at org.eclipse.jetty.ee10.servlet.ServletContextHandler.callContextDestroyed(ServletContextHandler.java:1608)
at org.eclipse.jetty.ee10.servlet.ServletContextHandler.contextDestroyed(ServletContextHandler.java:529)
at org.eclipse.jetty.ee10.servlet.ServletHandler.doStop(ServletHandler.java:329)
at org.eclipse.jetty.util.component.AbstractLifeCycle.stop(AbstractLifeCycle.java:132)
at org.eclipse.jetty.util.component.ContainerLifeCycle.stop(ContainerLifeCycle.java:180)
at org.eclipse.jetty.util.component.ContainerLifeCycle.doStop(ContainerLifeCycle.java:202)
at org.eclipse.jetty.server.Handler$Abstract.doStop(Handler.java:499)
at org.eclipse.jetty.server.handler.ContextHandler.lambda$doStop$0(ContextHandler.java:841)
at org.eclipse.jetty.server.handler.ContextHandler$ScopedContext.call(ContextHandler.java:1488)
at org.eclipse.jetty.server.handler.ContextHandler.doStop(ContextHandler.java:841)
at org.eclipse.jetty.ee10.servlet.ServletContextHandler.stopContext(ServletContextHandler.java:1334)
at org.eclipse.jetty.ee10.servlet.ServletContextHandler.doStop(ServletContextHandler.java:1085)
at org.eclipse.jetty.util.component.AbstractLifeCycle.stop(AbstractLifeCycle.java:132)
at org.eclipse.jetty.util.component.ContainerLifeCycle.stop(ContainerLifeCycle.java:180)
at org.eclipse.jetty.util.component.ContainerLifeCycle.doStop(ContainerLifeCycle.java:202)
at org.eclipse.jetty.server.Handler$Abstract.doStop(Handler.java:499)
at org.eclipse.jetty.util.component.AbstractLifeCycle.stop(AbstractLifeCycle.java:132)
at org.eclipse.jetty.util.component.ContainerLifeCycle.stop(ContainerLifeCycle.java:180)
at org.eclipse.jetty.util.component.ContainerLifeCycle.doStop(ContainerLifeCycle.java:202)
at org.eclipse.jetty.server.Handler$Abstract.doStop(Handler.java:499)
at org.eclipse.jetty.util.component.AbstractLifeCycle.stop(AbstractLifeCycle.java:132)
at org.eclipse.jetty.util.component.ContainerLifeCycle.stop(ContainerLifeCycle.java:180)
at org.eclipse.jetty.util.component.ContainerLifeCycle.doStop(ContainerLifeCycle.java:202)
at org.eclipse.jetty.server.Handler$Abstract.doStop(Handler.java:499)
at org.eclipse.jetty.server.Server.doStop(Server.java:690)
at org.eclipse.jetty.util.component.AbstractLifeCycle.stop(AbstractLifeCycle.java:132)
at org.apache.solr.embedded.JettySolrRunner.stop(JettySolrRunner.java:586)
at org.apache.solr.cloud.MiniSolrCloudCluster.stopJettySolrRunner(MiniSolrCloudCluster.java:527)
at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$shutdown$5(MiniSolrCloudCluster.java:627)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:408)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
... 1 more
[JENKINS] Solr-main-Linux (64bit/hotspot/jdk-22.0.2) - Build # 33531 - Still Unstable!
Posted by Policeman Jenkins Server via builds <bu...@solr.apache.org>.
Build: https://jenkins.thetaphi.de/job/Solr-main-Linux/33531/
Java: 64bit/hotspot/jdk-22.0.2 -XX:-UseCompressedOops -XX:+UseParallelGC
2 tests failed.
FAILED: org.apache.solr.cloud.LeaderTragicEventTest.classMethod
Error Message:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
at __randomizedtesting.SeedInfo.seed([F2BA0B3884086FA1]:0)
FAILED: org.apache.solr.search.join.TestScoreJoinQPNoScore.testJoin
Error Message:
java.lang.RuntimeException: mismatch: '10'!='13' @ response/docs/[0]/id
Stack Trace:
java.lang.RuntimeException: mismatch: '10'!='13' @ response/docs/[0]/id
at __randomizedtesting.SeedInfo.seed([F2BA0B3884086FA1:CF893EB2E55F6837]:0)
at org.apache.solr.SolrTestCaseJ4.assertJQ(SolrTestCaseJ4.java:985)
at org.apache.solr.SolrTestCaseJ4.assertJQ(SolrTestCaseJ4.java:931)
at org.apache.solr.search.join.TestScoreJoinQPNoScore.testJoin(TestScoreJoinQPNoScore.java:96)
at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103)
at java.base/java.lang.reflect.Method.invoke(Method.java:580)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1763)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48)
at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
at org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
at org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
at java.base/java.lang.Thread.run(Thread.java:1570)
[JENKINS-EA] Solr-main-Linux (64bit/hotspot/jdk-27-ea+14) - Build # 33530 - Still Unstable!
Posted by Policeman Jenkins Server via builds <bu...@solr.apache.org>.
Build: https://jenkins.thetaphi.de/job/Solr-main-Linux/33530/
Java: 64bit/hotspot/jdk-27-ea+14 -XX:-UseCompressedOops -XX:+UseZGC
3 tests failed.
FAILED: org.apache.solr.TestRandomDVFaceting.testRandomFaceting
Error Message:
org.apache.solr.common.SolrException: Error while processing facet fields: java.lang.OutOfMemoryError: Java heap space
Stack Trace:
org.apache.solr.common.SolrException: Error while processing facet fields: java.lang.OutOfMemoryError: Java heap space
at __randomizedtesting.SeedInfo.seed([779E0B00C49FB04D:7AF62BD5916678F2]:0)
at app//org.apache.solr.request.SimpleFacets.getFacetFieldCounts(SimpleFacets.java:952)
at app//org.apache.solr.handler.component.FacetComponent.getFacetCounts(FacetComponent.java:342)
at app//org.apache.solr.handler.component.FacetComponent.process(FacetComponent.java:278)
at app//org.apache.solr.handler.component.SearchHandler.processComponentsLocal(SearchHandler.java:486)
at app//org.apache.solr.handler.component.SearchHandler.processComponents(SearchHandler.java:414)
at app//org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:238)
at app//org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:251)
at app//org.apache.solr.core.SolrCore.execute(SolrCore.java:2919)
at app//org.apache.solr.util.TestHarness.query(TestHarness.java:330)
at app//org.apache.solr.util.TestHarness.query(TestHarness.java:310)
at app//org.apache.solr.TestRandomDVFaceting.doFacetTests(TestRandomDVFaceting.java:279)
at app//org.apache.solr.TestRandomDVFaceting.doFacetTests(TestRandomDVFaceting.java:183)
at app//org.apache.solr.TestRandomDVFaceting.testRandomFaceting(TestRandomDVFaceting.java:167)
at java.base@27-ea/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
at java.base@27-ea/java.lang.reflect.Method.invoke(Method.java:565)
at app//com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1763)
at app//com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
at app//com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
at app//com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
at app//org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48)
at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
at app//org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
at app//org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
at app//org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at app//org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at app//org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
at app//org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
at app//org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
at app//org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
at java.base@27-ea/java.lang.Thread.run(Thread.java:1527)
Caused by: java.lang.OutOfMemoryError: Java heap space
at org.apache.lucene.codecs.lucene104.Lucene104PostingsReader$BlockPostingsEnum.<init>(Lucene104PostingsReader.java:305)
at org.apache.lucene.codecs.lucene104.Lucene104PostingsReader.postings(Lucene104PostingsReader.java:263)
at org.apache.lucene.codecs.lucene103.blocktree.SegmentTermsEnum.postings(SegmentTermsEnum.java:1014)
at org.apache.lucene.index.FilterLeafReader$FilterTermsEnum.postings(FilterLeafReader.java:229)
at org.apache.lucene.index.MultiTermsEnum.postings(MultiTermsEnum.java:341)
at org.apache.solr.search.SolrIndexSearcher.getResult(SolrIndexSearcher.java:1420)
at org.apache.solr.search.SolrIndexSearcher.getDocSet(SolrIndexSearcher.java:1406)
at org.apache.solr.search.SolrIndexSearcher.numDocs(SolrIndexSearcher.java:2454)
at org.apache.solr.request.SimpleFacets.getFacetTermEnumCounts(SimpleFacets.java:1162)
at org.apache.solr.request.SimpleFacets.getTermCounts(SimpleFacets.java:526)
at org.apache.solr.request.SimpleFacets.getTermCounts(SimpleFacets.java:428)
at org.apache.solr.request.SimpleFacets.lambda$getFacetFieldCounts$0(SimpleFacets.java:915)
at org.apache.solr.request.SimpleFacets$$Lambda/0x000000000db06bb8.call(Unknown Source)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:330)
at org.apache.solr.request.SimpleFacets$3.execute(SimpleFacets.java:860)
at org.apache.solr.request.SimpleFacets.getFacetFieldCounts(SimpleFacets.java:930)
at org.apache.solr.handler.component.FacetComponent.getFacetCounts(FacetComponent.java:342)
at org.apache.solr.handler.component.FacetComponent.process(FacetComponent.java:278)
at org.apache.solr.handler.component.SearchHandler.processComponentsLocal(SearchHandler.java:486)
at org.apache.solr.handler.component.SearchHandler.processComponents(SearchHandler.java:414)
at org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:238)
at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:251)
at org.apache.solr.core.SolrCore.execute(SolrCore.java:2919)
at org.apache.solr.util.TestHarness.query(TestHarness.java:330)
at org.apache.solr.util.TestHarness.query(TestHarness.java:310)
at org.apache.solr.TestRandomDVFaceting.doFacetTests(TestRandomDVFaceting.java:279)
at org.apache.solr.TestRandomDVFaceting.doFacetTests(TestRandomDVFaceting.java:183)
at org.apache.solr.TestRandomDVFaceting.testRandomFaceting(TestRandomDVFaceting.java:167)
at java.base/java.lang.invoke.LambdaForm$DMH/0x000000000d124000.invokeVirtual(LambdaForm$DMH)
at java.base/java.lang.invoke.LambdaForm$MH/0x000000000d098800.invoke(LambdaForm$MH)
at java.base/java.lang.invoke.Invokers$Holder.invokeExact_MT(Invokers$Holder)
at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invokeImpl(DirectMethodHandleAccessor.java:154)
FAILED: org.apache.solr.cloud.api.collections.LocalFSCloudIncrementalBackupTest.testCustomProperties
Error Message:
org.apache.solr.common.SolrException: Could not find a healthy node to handle the request.
Stack Trace:
org.apache.solr.common.SolrException: Could not find a healthy node to handle the request.
at __randomizedtesting.SeedInfo.seed([779E0B00C49FB04D:65C548D1C5EAC498]:0)
at app//org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1548)
at app//org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1190)
at app//org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1375)
at app//org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1375)
at app//org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1375)
at app//org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1366)
at app//org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1351)
at app//org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:1112)
at app//org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:302)
at app//org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:927)
at app//org.apache.solr.cloud.api.collections.LocalFSCloudIncrementalBackupTest.testCustomProperties(LocalFSCloudIncrementalBackupTest.java:155)
at java.base@27-ea/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
at java.base@27-ea/java.lang.reflect.Method.invoke(Method.java:565)
at app//com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1763)
at app//com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
at app//com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
at app//com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
at app//org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48)
at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
at app//org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
at app//org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
at app//org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
at app//com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
at app//com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
at app//com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
at app//com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
at app//com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at app//org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at app//com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at app//org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at app//org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
at app//org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
at app//org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
at app//org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
at app//org.junit.rules.RunRules.evaluate(RunRules.java:20)
at app//com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at app//com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
at app//com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
at java.base@27-ea/java.lang.Thread.run(Thread.java:1527)
FAILED: org.apache.solr.crossdc.manager.SolrAndKafkaIntegrationTest.testPartitioning
Error Message:
java.util.ConcurrentModificationException
Stack Trace:
java.util.ConcurrentModificationException
at __randomizedtesting.SeedInfo.seed([779E0B00C49FB04D:2B8F19CAC0757A9A]:0)
at java.base/java.util.ArrayList$Itr.checkForComodification(ArrayList.java:1104)
at java.base/java.util.ArrayList$Itr.next(ArrayList.java:1058)
at org.apache.solr.crossdc.manager.SolrAndKafkaIntegrationTest.testPartitioning(SolrAndKafkaIntegrationTest.java:363)
at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104)
at java.base/java.lang.reflect.Method.invoke(Method.java:565)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1763)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:946)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:982)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at org.apache.lucene.tests.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:48)
at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
at org.apache.lucene.tests.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:45)
at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:843)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:490)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:955)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:840)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:891)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
at org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
at org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
at org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
at java.base/java.lang.Thread.run(Thread.java:1527)