You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@solr.apache.org by Kevin Risden <kr...@apache.org> on 2022/10/16 00:00:26 UTC

Re: [JENKINS] Solr-main-Linux (64bit/jdk-16.0.2) - Build # 7849 - Still Unstable!

Most likely one of my commits today caused this. Apparently I didn't run
the hdfs tests... My guess is commons-text or Gradle. I won't be able to
look at this until tomorrow at the earliest.

Kevin Risden

On Sat, Oct 15, 2022, 19:55 Policeman Jenkins Server <je...@thetaphi.de>
wrote:

> Build: https://jenkins.thetaphi.de/job/Solr-main-Linux/7849/
> Java: 64bit/jdk-16.0.2 -XX:-UseCompressedOops -XX:+UseSerialGC
>
> 12 tests failed.
> FAILED:
> org.apache.solr.hdfs.cloud.api.collections.HdfsCloudIncrementalBackupTest.classMethod
>
> Error Message:
> java.lang.Exception: Error starting up MiniSolrCloudCluster
>
> Stack Trace:
> java.lang.Exception: Error starting up MiniSolrCloudCluster
>         at __randomizedtesting.SeedInfo.seed([183F02B0C82BE2D2]:0)
>         at
> org.apache.solr.cloud.MiniSolrCloudCluster.checkForExceptions(MiniSolrCloudCluster.java:767)
>         at
> org.apache.solr.cloud.MiniSolrCloudCluster.<init>(MiniSolrCloudCluster.java:362)
>         at
> org.apache.solr.cloud.MiniSolrCloudCluster$Builder.build(MiniSolrCloudCluster.java:1233)
>         at
> org.apache.solr.cloud.MiniSolrCloudCluster$Builder.configure(MiniSolrCloudCluster.java:1205)
>         at
> org.apache.solr.hdfs.cloud.api.collections.HdfsCloudIncrementalBackupTest.setupClass(HdfsCloudIncrementalBackupTest.java:129)
>         at
> java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native
> Method)
>         at
> java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:78)
>         at
> java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>         at java.base/java.lang.reflect.Method.invoke(Method.java:567)
>         at
> com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
>         at
> com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:886)
>         at
> com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
>         at
> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at
> org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
>         at
> org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
>         at
> com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
>         at org.junit.rules.RunRules.evaluate(RunRules.java:20)
>         at
> org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
>         at
> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at
> org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
>         at
> com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
>         at
> com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
>         at
> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at
> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at
> org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
>         at
> org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
>         at
> org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
>         at
> org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
>         at
> org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
>         at org.junit.rules.RunRules.evaluate(RunRules.java:20)
>         at
> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at
> com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
>         at
> com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
>         at java.base/java.lang.Thread.run(Thread.java:831)
>         Suppressed: java.lang.NoClassDefFoundError: Could not initialize
> class org.apache.hadoop.shaded.com
> .sun.xml.bind.v2.runtime.reflect.opt.Injector
>                 at org.apache.hadoop.shaded.com
> .sun.xml.bind.v2.runtime.reflect.opt.AccessorInjector.prepare(AccessorInjector.java:83)
>                 at org.apache.hadoop.shaded.com
> .sun.xml.bind.v2.runtime.reflect.opt.OptimizedAccessorFactory.get(OptimizedAccessorFactory.java:176)
>                 at org.apache.hadoop.shaded.com
> .sun.xml.bind.v2.runtime.reflect.Accessor$FieldReflection.optimize(Accessor.java:282)
>                 at org.apache.hadoop.shaded.com
> .sun.xml.bind.v2.runtime.property.ArrayProperty.<init>(ArrayProperty.java:69)
>                 at org.apache.hadoop.shaded.com
> .sun.xml.bind.v2.runtime.property.ArrayERProperty.<init>(ArrayERProperty.java:88)
>                 at org.apache.hadoop.shaded.com
> .sun.xml.bind.v2.runtime.property.ArrayElementProperty.<init>(ArrayElementProperty.java:100)
>                 at org.apache.hadoop.shaded.com
> .sun.xml.bind.v2.runtime.property.ArrayElementNodeProperty.<init>(ArrayElementNodeProperty.java:62)
>                 at
> java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native
> Method)
>                 at
> java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:78)
>                 at
> java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
>                 at
> java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499)
>                 at
> java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480)
>                 at org.apache.hadoop.shaded.com
> .sun.xml.bind.v2.runtime.property.PropertyFactory.create(PropertyFactory.java:128)
>                 at org.apache.hadoop.shaded.com
> .sun.xml.bind.v2.runtime.ClassBeanInfoImpl.<init>(ClassBeanInfoImpl.java:183)
>                 at org.apache.hadoop.shaded.com
> .sun.xml.bind.v2.runtime.JAXBContextImpl.getOrCreate(JAXBContextImpl.java:532)
>                 at org.apache.hadoop.shaded.com
> .sun.xml.bind.v2.runtime.JAXBContextImpl.<init>(JAXBContextImpl.java:347)
>                 at org.apache.hadoop.shaded.com
> .sun.xml.bind.v2.runtime.JAXBContextImpl$JAXBContextBuilder.build(JAXBContextImpl.java:1170)
>                 at org.apache.hadoop.shaded.com
> .sun.xml.bind.v2.ContextFactory.createContext(ContextFactory.java:145)
>                 at org.apache.hadoop.shaded.com
> .sun.xml.bind.v2.ContextFactory.createContext(ContextFactory.java:236)
>                 at
> java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native
> Method)
>                 at
> java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:78)
>                 at
> java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>                 at
> java.base/java.lang.reflect.Method.invoke(Method.java:567)
>                 at
> javax.xml.bind.ContextFinder.newInstance(ContextFinder.java:171)
>                 at
> javax.xml.bind.ContextFinder.newInstance(ContextFinder.java:129)
>                 at
> javax.xml.bind.ContextFinder.find(ContextFinder.java:307)
>                 at
> javax.xml.bind.JAXBContext.newInstance(JAXBContext.java:478)
>                 at
> javax.xml.bind.JAXBContext.newInstance(JAXBContext.java:435)
>                 at
> org.glassfish.jersey.server.wadl.internal.WadlApplicationContextImpl.getJAXBContextFromWadlGenerator(WadlApplicationContextImpl.java:121)
>                 at
> org.glassfish.jersey.server.wadl.internal.WadlApplicationContextImpl.isJaxbImplAvailable(WadlApplicationContextImpl.java:270)
>                 at
> org.glassfish.jersey.server.wadl.WadlFeature.configure(WadlFeature.java:65)
>                 at
> org.glassfish.jersey.model.internal.CommonConfig.configureFeatures(CommonConfig.java:728)
>                 at
> org.glassfish.jersey.model.internal.CommonConfig.configureMetaProviders(CommonConfig.java:647)
>                 at
> org.glassfish.jersey.server.ResourceConfig.configureMetaProviders(ResourceConfig.java:824)
>                 at
> org.glassfish.jersey.server.ApplicationHandler.initialize(ApplicationHandler.java:332)
>                 at
> org.glassfish.jersey.server.ApplicationHandler.lambda$initialize$1(ApplicationHandler.java:297)
>                 at
> org.glassfish.jersey.internal.Errors.process(Errors.java:292)
>                 at
> org.glassfish.jersey.internal.Errors.process(Errors.java:274)
>                 at
> org.glassfish.jersey.internal.Errors.processWithException(Errors.java:232)
>                 at
> org.glassfish.jersey.server.ApplicationHandler.initialize(ApplicationHandler.java:296)
>                 at
> org.glassfish.jersey.server.ApplicationHandler.<init>(ApplicationHandler.java:261)
>                 at
> org.glassfish.jersey.server.ApplicationHandler.<init>(ApplicationHandler.java:236)
>                 at
> org.apache.solr.core.CoreContainer.load(CoreContainer.java:1086)
>                 at
> org.apache.solr.servlet.CoreContainerProvider.createCoreContainer(CoreContainerProvider.java:377)
>                 at
> org.apache.solr.servlet.CoreContainerProvider.init(CoreContainerProvider.java:226)
>                 at
> org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:402)
>                 at
> org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:194)
>                 at
> org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:74)
>                 at
> org.apache.solr.client.solrj.embedded.JettySolrRunner.retryOnPortBindFailure(JettySolrRunner.java:614)
>                 at
> org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:552)
>                 at
> org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:523)
>                 at
> org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:550)
>                 at
> org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:355)
>                 at
> java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
>                 at
> org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:271)
>                 at
> java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
>                 at
> java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
>                 ... 1 more
>         Suppressed: java.lang.ExceptionInInitializerError
>                 at org.apache.hadoop.shaded.com
> .sun.xml.bind.v2.runtime.reflect.opt.AccessorInjector.prepare(AccessorInjector.java:83)
>                 at org.apache.hadoop.shaded.com
> .sun.xml.bind.v2.runtime.reflect.opt.OptimizedAccessorFactory.get(OptimizedAccessorFactory.java:176)
>                 at org.apache.hadoop.shaded.com
> .sun.xml.bind.v2.runtime.reflect.Accessor$FieldReflection.optimize(Accessor.java:282)
>                 at org.apache.hadoop.shaded.com
> .sun.xml.bind.v2.runtime.property.ArrayProperty.<init>(ArrayProperty.java:69)
>                 at org.apache.hadoop.shaded.com
> .sun.xml.bind.v2.runtime.property.ArrayERProperty.<init>(ArrayERProperty.java:88)
>                 at org.apache.hadoop.shaded.com
> .sun.xml.bind.v2.runtime.property.ArrayElementProperty.<init>(ArrayElementProperty.java:100)
>                 at org.apache.hadoop.shaded.com
> .sun.xml.bind.v2.runtime.property.ArrayElementNodeProperty.<init>(ArrayElementNodeProperty.java:62)
>                 at
> java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native
> Method)
>                 at
> java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:78)
>                 at
> java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
>                 at
> java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499)
>                 at
> java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480)
>                 at org.apache.hadoop.shaded.com
> .sun.xml.bind.v2.runtime.property.PropertyFactory.create(PropertyFactory.java:128)
>                 at org.apache.hadoop.shaded.com
> .sun.xml.bind.v2.runtime.ClassBeanInfoImpl.<init>(ClassBeanInfoImpl.java:183)
>                 at org.apache.hadoop.shaded.com
> .sun.xml.bind.v2.runtime.JAXBContextImpl.getOrCreate(JAXBContextImpl.java:532)
>                 at org.apache.hadoop.shaded.com
> .sun.xml.bind.v2.runtime.JAXBContextImpl.<init>(JAXBContextImpl.java:347)
>                 at org.apache.hadoop.shaded.com
> .sun.xml.bind.v2.runtime.JAXBContextImpl$JAXBContextBuilder.build(JAXBContextImpl.java:1170)
>                 at org.apache.hadoop.shaded.com
> .sun.xml.bind.v2.ContextFactory.createContext(ContextFactory.java:145)
>                 at org.apache.hadoop.shaded.com
> .sun.xml.bind.v2.ContextFactory.createContext(ContextFactory.java:236)
>                 at
> java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native
> Method)
>                 at
> java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:78)
>                 at
> java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>                 at
> java.base/java.lang.reflect.Method.invoke(Method.java:567)
>                 at
> javax.xml.bind.ContextFinder.newInstance(ContextFinder.java:171)
>                 at
> javax.xml.bind.ContextFinder.newInstance(ContextFinder.java:129)
>                 at
> javax.xml.bind.ContextFinder.find(ContextFinder.java:307)
>                 at
> javax.xml.bind.JAXBContext.newInstance(JAXBContext.java:478)
>                 at
> javax.xml.bind.JAXBContext.newInstance(JAXBContext.java:435)
>                 at
> org.glassfish.jersey.server.wadl.internal.WadlApplicationContextImpl.getJAXBContextFromWadlGenerator(WadlApplicationContextImpl.java:121)
>                 at
> org.glassfish.jersey.server.wadl.internal.WadlApplicationContextImpl.isJaxbImplAvailable(WadlApplicationContextImpl.java:270)
>                 at
> org.glassfish.jersey.server.wadl.WadlFeature.configure(WadlFeature.java:65)
>                 at
> org.glassfish.jersey.model.internal.CommonConfig.configureFeatures(CommonConfig.java:728)
>                 at
> org.glassfish.jersey.model.internal.CommonConfig.configureMetaProviders(CommonConfig.java:647)
>                 at
> org.glassfish.jersey.server.ResourceConfig.configureMetaProviders(ResourceConfig.java:824)
>                 at
> org.glassfish.jersey.server.ApplicationHandler.initialize(ApplicationHandler.java:332)
>                 at
> org.glassfish.jersey.server.ApplicationHandler.lambda$initialize$1(ApplicationHandler.java:297)
>                 at
> org.glassfish.jersey.internal.Errors.process(Errors.java:292)
>                 at
> org.glassfish.jersey.internal.Errors.process(Errors.java:274)
>                 at
> org.glassfish.jersey.internal.Errors.processWithException(Errors.java:232)
>                 at
> org.glassfish.jersey.server.ApplicationHandler.initialize(ApplicationHandler.java:296)
>                 at
> org.glassfish.jersey.server.ApplicationHandler.<init>(ApplicationHandler.java:261)
>                 at
> org.glassfish.jersey.server.ApplicationHandler.<init>(ApplicationHandler.java:236)
>                 at
> org.apache.solr.core.CoreContainer.load(CoreContainer.java:1086)
>                 at
> org.apache.solr.servlet.CoreContainerProvider.createCoreContainer(CoreContainerProvider.java:377)
>                 at
> org.apache.solr.servlet.CoreContainerProvider.init(CoreContainerProvider.java:226)
>                 at
> org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:402)
>                 at
> org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:194)
>                 at
> org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:74)
>                 at
> org.apache.solr.client.solrj.embedded.JettySolrRunner.retryOnPortBindFailure(JettySolrRunner.java:614)
>                 at
> org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:552)
>                 at
> org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:523)
>                 at
> org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:550)
>                 at
> org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:355)
>                 at
> java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
>                 at
> org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:271)
>                 at
> java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
>                 at
> java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
>                 ... 1 more
>         Caused by: java.lang.reflect.InaccessibleObjectException: Unable
> to make protected final java.lang.Class
> java.lang.ClassLoader.defineClass(java.lang.String,byte[],int,int) throws
> java.lang.ClassFormatError accessible: module java.base does not "opens
> java.lang" to unnamed module @77f1baf5
>                 at
> java.base/java.lang.reflect.AccessibleObject.checkCanSetAccessible(AccessibleObject.java:357)
>                 at
> java.base/java.lang.reflect.AccessibleObject.checkCanSetAccessible(AccessibleObject.java:297)
>                 at
> java.base/java.lang.reflect.Method.checkCanSetAccessible(Method.java:199)
>                 at
> java.base/java.lang.reflect.Method.setAccessible(Method.java:193)
>                 at org.apache.hadoop.shaded.com
> .sun.xml.bind.v2.runtime.reflect.opt.Injector$1.run(Injector.java:177)
>                 at org.apache.hadoop.shaded.com
> .sun.xml.bind.v2.runtime.reflect.opt.Injector$1.run(Injector.java:174)
>                 at
> java.base/java.security.AccessController.doPrivileged(AccessController.java:312)
>                 at org.apache.hadoop.shaded.com
> .sun.xml.bind.v2.runtime.reflect.opt.Injector.<clinit>(Injector.java:172)
>                 ... 58 more
>
>
> FAILED:
> org.apache.solr.hdfs.cloud.api.collections.HdfsCloudIncrementalBackupTest.classMethod
>
> Error Message:
> com.carrotsearch.randomizedtesting.ThreadLeakError: 46 threads leaked from
> SUITE scope at
> org.apache.solr.hdfs.cloud.api.collections.HdfsCloudIncrementalBackupTest:
>    1) Thread[id=401, name=CloserThread, state=WAITING,
> group=TGRP-HdfsCloudIncrementalBackupTest]
>         at java.base@16.0.2/java.lang.Object.wait(Native Method)
>         at java.base@16.0.2/java.lang.Object.wait(Object.java:320)
>         at
> app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2506)
>    2) Thread[id=372, name=Session-HouseKeeper-74f5e0-1,
> state=TIMED_WAITING, group=TGRP-HdfsCloudIncrementalBackupTest]
>         at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
>         at java.base@16.0.2
> /java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
>         at java.base@16.0.2
> /java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
>         at java.base@16.0.2
> /java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
>         at java.base@16.0.2
> /java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
>         at java.base@16.0.2
> /java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
>         at java.base@16.0.2
> /java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
>         at java.base@16.0.2
> /java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>    3) Thread[id=400,
> name=OverseerCollectionConfigSetProcessor-72262794004201478-127.0.0.1:46753_solr-n_0000000000,
> state=WAITING, group=Overseer collection creation process.]
>         at java.base@16.0.2/java.lang.Object.wait(Native Method)
>         at java.base@16.0.2/java.lang.Object.wait(Object.java:320)
>         at
> app//org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1590)
>         at
> app//org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1562)
>         at app//org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1970)
>         at
> app//org.apache.solr.common.cloud.SolrZkClient.lambda$getData$6(SolrZkClient.java:432)
>         at
> app//org.apache.solr.common.cloud.SolrZkClient$$Lambda$382/0x00000008013d7858.execute(Unknown
> Source)
>         at
> app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:72)
>         at
> app//org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:432)
>         at
> app//org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:486)
>         at
> app//org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:232)
>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>    4) Thread[id=367, name=qtp1790869424-367, state=RUNNABLE,
> group=TGRP-HdfsCloudIncrementalBackupTest]
>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>         at java.base@16.0.2/sun.nio.ch
> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>         at java.base@16.0.2/sun.nio.ch
> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>         at java.base@16.0.2/sun.nio.ch
> .SelectorImpl.select(SelectorImpl.java:146)
>         at app//org.eclipse.jetty.io
> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>         at app//org.eclipse.jetty.io
> .ManagedSelector.select(ManagedSelector.java:190)
>         at app//org.eclipse.jetty.io
> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>         at app//org.eclipse.jetty.io
> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>         at
> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>         at
> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>         at
> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>         at
> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
> Source)
>         at
> app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
>         at
> app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>    5) Thread[id=360, name=qtp577348919-360, state=RUNNABLE,
> group=TGRP-HdfsCloudIncrementalBackupTest]
>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>         at java.base@16.0.2/sun.nio.ch
> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>         at java.base@16.0.2/sun.nio.ch
> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>         at java.base@16.0.2/sun.nio.ch
> .SelectorImpl.select(SelectorImpl.java:146)
>         at app//org.eclipse.jetty.io
> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>         at app//org.eclipse.jetty.io
> .ManagedSelector.select(ManagedSelector.java:190)
>         at app//org.eclipse.jetty.io
> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>         at app//org.eclipse.jetty.io
> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>         at
> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>         at
> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>         at
> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>         at
> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
> Source)
>         at
> app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
>         at
> app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>    6) Thread[id=404, name=coreContainerWorkExecutor-18-thread-1,
> state=TIMED_WAITING, group=TGRP-HdfsCloudIncrementalBackupTest]
>         at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
>         at java.base@16.0.2
> /java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
>         at java.base@16.0.2
> /java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
>         at java.base@16.0.2
> /java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
>         at java.base@16.0.2
> /java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
>         at java.base@16.0.2
> /java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
>         at java.base@16.0.2
> /java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
>         at java.base@16.0.2
> /java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>    7) Thread[id=383, name=httpShardExecutor-21-thread-1, state=RUNNABLE,
> group=TGRP-HdfsCloudIncrementalBackupTest]
>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>         at java.base@16.0.2/sun.nio.ch
> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>         at java.base@16.0.2/sun.nio.ch
> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>         at java.base@16.0.2/sun.nio.ch
> .SelectorImpl.select(SelectorImpl.java:146)
>         at app//org.eclipse.jetty.io
> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>         at app//org.eclipse.jetty.io
> .ManagedSelector.select(ManagedSelector.java:190)
>         at app//org.eclipse.jetty.io
> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>         at app//org.eclipse.jetty.io
> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>         at
> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>         at
> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>         at
> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>         at
> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
> Source)
>         at
> app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:271)
>         at
> app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$349/0x0000000801384860.run(Unknown
> Source)
>         at java.base@16.0.2
> /java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
>         at java.base@16.0.2
> /java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>    8) Thread[id=364, name=qtp1790869424-364, state=RUNNABLE,
> group=TGRP-HdfsCloudIncrementalBackupTest]
>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>         at java.base@16.0.2/sun.nio.ch
> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>         at java.base@16.0.2/sun.nio.ch
> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>         at java.base@16.0.2/sun.nio.ch
> .SelectorImpl.select(SelectorImpl.java:146)
>         at app//org.eclipse.jetty.io
> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>         at app//org.eclipse.jetty.io
> .ManagedSelector.select(ManagedSelector.java:190)
>         at app//org.eclipse.jetty.io
> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>         at app//org.eclipse.jetty.io
> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>         at
> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>         at
> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>         at
> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>         at
> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
> Source)
>         at
> app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
>         at
> app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>    9) Thread[id=428, name=zkCallback-32-thread-4, state=TIMED_WAITING,
> group=TGRP-HdfsCloudIncrementalBackupTest]
>         at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
>         at java.base@16.0.2
> /java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
>         at java.base@16.0.2
> /java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
>         at java.base@16.0.2
> /java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
>         at java.base@16.0.2
> /java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
>         at java.base@16.0.2
> /java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
>         at java.base@16.0.2
> /java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
>         at java.base@16.0.2
> /java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>   10) Thread[id=418, name=zkCallback-32-thread-2, state=TIMED_WAITING,
> group=TGRP-HdfsCloudIncrementalBackupTest]
>         at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
>         at java.base@16.0.2
> /java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
>         at java.base@16.0.2
> /java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
>         at java.base@16.0.2
> /java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
>         at java.base@16.0.2
> /java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
>         at java.base@16.0.2
> /java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
>         at java.base@16.0.2
> /java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
>         at java.base@16.0.2
> /java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>   11) Thread[id=365, name=qtp577348919-365, state=RUNNABLE,
> group=TGRP-HdfsCloudIncrementalBackupTest]
>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>         at java.base@16.0.2/sun.nio.ch
> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>         at java.base@16.0.2/sun.nio.ch
> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>         at java.base@16.0.2/sun.nio.ch
> .SelectorImpl.select(SelectorImpl.java:146)
>         at app//org.eclipse.jetty.io
> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>         at app//org.eclipse.jetty.io
> .ManagedSelector.select(ManagedSelector.java:190)
>         at app//org.eclipse.jetty.io
> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>         at app//org.eclipse.jetty.io
> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>         at
> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>         at
> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>         at
> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>         at
> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
> Source)
>         at
> app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
>         at
> app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>   12) Thread[id=369, name=qtp1790869424-369, state=RUNNABLE,
> group=TGRP-HdfsCloudIncrementalBackupTest]
>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>         at java.base@16.0.2/sun.nio.ch
> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>         at java.base@16.0.2/sun.nio.ch
> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>         at java.base@16.0.2/sun.nio.ch
> .SelectorImpl.select(SelectorImpl.java:146)
>         at app//org.eclipse.jetty.io
> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>         at app//org.eclipse.jetty.io
> .ManagedSelector.select(ManagedSelector.java:190)
>         at app//org.eclipse.jetty.io
> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>         at app//org.eclipse.jetty.io
> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>         at
> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>         at
> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>         at
> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>         at
> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
> Source)
>         at
> app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
>         at
> app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>   13) Thread[id=387, name=httpShardExecutor-26-thread-1, state=RUNNABLE,
> group=TGRP-HdfsCloudIncrementalBackupTest]
>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>         at java.base@16.0.2/sun.nio.ch
> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>         at java.base@16.0.2/sun.nio.ch
> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>         at java.base@16.0.2/sun.nio.ch
> .SelectorImpl.select(SelectorImpl.java:146)
>         at app//org.eclipse.jetty.io
> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>         at app//org.eclipse.jetty.io
> .ManagedSelector.select(ManagedSelector.java:190)
>         at app//org.eclipse.jetty.io
> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>         at app//org.eclipse.jetty.io
> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>         at
> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>         at
> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>         at
> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>         at
> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
> Source)
>         at
> app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:271)
>         at
> app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$349/0x0000000801384860.run(Unknown
> Source)
>         at java.base@16.0.2
> /java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
>         at java.base@16.0.2
> /java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>   14) Thread[id=355, name=qtp577348919-355, state=RUNNABLE,
> group=TGRP-HdfsCloudIncrementalBackupTest]
>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>         at java.base@16.0.2/sun.nio.ch
> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>         at java.base@16.0.2/sun.nio.ch
> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>         at java.base@16.0.2/sun.nio.ch
> .SelectorImpl.select(SelectorImpl.java:146)
>         at app//org.eclipse.jetty.io
> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>         at app//org.eclipse.jetty.io
> .ManagedSelector.select(ManagedSelector.java:190)
>         at app//org.eclipse.jetty.io
> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>         at app//org.eclipse.jetty.io
> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>         at
> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>         at
> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>         at
> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>         at
> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
> Source)
>         at
> app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
>         at
> app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>   15) Thread[id=396, name=zkConnectionManagerCallback-33-thread-1,
> state=WAITING, group=TGRP-HdfsCloudIncrementalBackupTest]
>         at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
>         at java.base@16.0.2
> /java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
>         at java.base@16.0.2
> /java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
>         at java.base@16.0.2
> /java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
>         at java.base@16.0.2
> /java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
>         at java.base@16.0.2
> /java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
>         at java.base@16.0.2
> /java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
>         at java.base@16.0.2
> /java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
>         at java.base@16.0.2
> /java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>   16) Thread[id=362, name=qtp577348919-362, state=RUNNABLE,
> group=TGRP-HdfsCloudIncrementalBackupTest]
>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>         at java.base@16.0.2/sun.nio.ch
> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>         at java.base@16.0.2/sun.nio.ch
> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>         at java.base@16.0.2/sun.nio.ch
> .SelectorImpl.select(SelectorImpl.java:146)
>         at app//org.eclipse.jetty.io
> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>         at app//org.eclipse.jetty.io
> .ManagedSelector.select(ManagedSelector.java:190)
>         at app//org.eclipse.jetty.io
> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>         at app//org.eclipse.jetty.io
> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>         at
> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>         at
> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>         at
> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>         at
> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
> Source)
>         at
> app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
>         at
> app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>   17) Thread[id=363, name=qtp577348919-363, state=RUNNABLE,
> group=TGRP-HdfsCloudIncrementalBackupTest]
>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>         at java.base@16.0.2/sun.nio.ch
> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>         at java.base@16.0.2/sun.nio.ch
> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>         at java.base@16.0.2/sun.nio.ch
> .SelectorImpl.select(SelectorImpl.java:146)
>         at app//org.eclipse.jetty.io
> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>         at app//org.eclipse.jetty.io
> .ManagedSelector.select(ManagedSelector.java:190)
>         at app//org.eclipse.jetty.io
> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>         at app//org.eclipse.jetty.io
> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>         at
> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>         at
> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>         at
> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>         at
> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
> Source)
>         at
> app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
>         at
> app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>   18) Thread[id=389, name=jetty-launcher-9-thread-2-EventThread,
> state=WAITING, group=TGRP-HdfsCloudIncrementalBackupTest]
>         at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
>         at java.base@16.0.2
> /java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
>         at java.base@16.0.2
> /java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
>         at java.base@16.0.2
> /java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
>         at java.base@16.0.2
> /java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
>         at java.base@16.0.2
> /java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
>         at
> app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:549)
>   19) Thread[id=374,
> name=qtp1790869424-374-acceptor-0@31e11e51-ServerConnector@442851d5{ssl,
> (ssl, alpn, http/1.1, h2)}{127.0.0.1:44953}, state=RUNNABLE,
> group=TGRP-HdfsCloudIncrementalBackupTest]
>         at java.base@16.0.2/sun.nio.ch.Net.accept(Native Method)
>         at java.base@16.0.2/sun.nio.ch
> .ServerSocketChannelImpl.implAccept(ServerSocketChannelImpl.java:424)
>         at java.base@16.0.2/sun.nio.ch
> .ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:390)
>         at
> app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388)
>         at
> app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:704)
>         at
> app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
>         at
> app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>   20) Thread[id=356, name=qtp1790869424-356, state=RUNNABLE,
> group=TGRP-HdfsCloudIncrementalBackupTest]
>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>         at java.base@16.0.2/sun.nio.ch
> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>         at java.base@16.0.2/sun.nio.ch
> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>         at java.base@16.0.2/sun.nio.ch
> .SelectorImpl.select(SelectorImpl.java:146)
>         at app//org.eclipse.jetty.io
> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>         at app//org.eclipse.jetty.io
> .ManagedSelector.select(ManagedSelector.java:190)
>         at app//org.eclipse.jetty.io
> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>         at app//org.eclipse.jetty.io
> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>         at
> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>         at
> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>         at
> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>         at
> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
> Source)
>         at
> app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
>         at
> app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>   21) Thread[id=417, name=zkCallback-27-thread-2, state=TIMED_WAITING,
> group=TGRP-HdfsCloudIncrementalBackupTest]
>         at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
>         at java.base@16.0.2
> /java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
>         at java.base@16.0.2
> /java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
>         at java.base@16.0.2
> /java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
>         at java.base@16.0.2
> /java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
>         at java.base@16.0.2
> /java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
>         at java.base@16.0.2
> /java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
>         at java.base@16.0.2
> /java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>   22) Thread[id=359, name=qtp577348919-359, state=RUNNABLE,
> group=TGRP-HdfsCloudIncrementalBackupTest]
>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>         at java.base@16.0.2/sun.nio.ch
> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>         at java.base@16.0.2/sun.nio.ch
> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>         at java.base@16.0.2/sun.nio.ch
> .SelectorImpl.select(SelectorImpl.java:146)
>         at app//org.eclipse.jetty.io
> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>         at app//org.eclipse.jetty.io
> .ManagedSelector.select(ManagedSelector.java:190)
>         at app//org.eclipse.jetty.io
> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>         at app//org.eclipse.jetty.io
> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>         at
> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>         at
> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>         at
> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>         at
> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
> Source)
>         at
> app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
>         at
> app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>   23) Thread[id=386, name=h2sc-23-thread-1, state=RUNNABLE,
> group=TGRP-HdfsCloudIncrementalBackupTest]
>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>         at java.base@16.0.2/sun.nio.ch
> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>         at java.base@16.0.2/sun.nio.ch
> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>         at java.base@16.0.2/sun.nio.ch
> .SelectorImpl.select(SelectorImpl.java:146)
>         at app//org.eclipse.jetty.io
> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>         at app//org.eclipse.jetty.io
> .ManagedSelector.select(ManagedSelector.java:190)
>         at app//org.eclipse.jetty.io
> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>         at app//org.eclipse.jetty.io
> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>         at
> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>         at
> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>         at
> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>         at
> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
> Source)
>         at
> app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:271)
>         at
> app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$349/0x0000000801384860.run(Unknown
> Source)
>         at java.base@16.0.2
> /java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
>         at java.base@16.0.2
> /java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>   24) Thread[id=366, name=qtp1790869424-366, state=RUNNABLE,
> group=TGRP-HdfsCloudIncrementalBackupTest]
>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>         at java.base@16.0.2/sun.nio.ch
> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>         at java.base@16.0.2/sun.nio.ch
> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>         at java.base@16.0.2/sun.nio.ch
> .SelectorImpl.select(SelectorImpl.java:146)
>         at app//org.eclipse.jetty.io
> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>         at app//org.eclipse.jetty.io
> .ManagedSelector.select(ManagedSelector.java:190)
>         at app//org.eclipse.jetty.io
> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>         at app//org.eclipse.jetty.io
> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>         at
> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>         at
> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>         at
> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>         at
> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
> Source)
>         at
> app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
>         at
> app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>   25) Thread[id=402, name=CloserThread, state=WAITING,
> group=TGRP-HdfsCloudIncrementalBackupTest]
>         at java.base@16.0.2/java.lang.Object.wait(Native Method)
>         at java.base@16.0.2/java.lang.Object.wait(Object.java:320)
>         at
> app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2506)
>   26) Thread[id=361, name=qtp1790869424-361, state=RUNNABLE,
> group=TGRP-HdfsCloudIncrementalBackupTest]
>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>         at java.base@16.0.2/sun.nio.ch
> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>         at java.base@16.0.2/sun.nio.ch
> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>         at java.base@16.0.2/sun.nio.ch
> .SelectorImpl.select(SelectorImpl.java:146)
>         at app//org.eclipse.jetty.io
> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>         at app//org.eclipse.jetty.io
> .ManagedSelector.select(ManagedSelector.java:190)
>         at app//org.eclipse.jetty.io
> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>         at app//org.eclipse.jetty.io
> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>         at
> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>         at
> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>         at
> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>         at
> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
> Source)
>         at
> app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
>         at
> app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>   27) Thread[id=393, name=h2sc-30-thread-1, state=RUNNABLE,
> group=TGRP-HdfsCloudIncrementalBackupTest]
>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>         at java.base@16.0.2/sun.nio.ch
> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>         at java.base@16.0.2/sun.nio.ch
> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>

Re: [JENKINS] Solr-main-Linux (64bit/jdk-16.0.2) - Build # 7849 - Still Unstable!

Posted by Kevin Risden <kr...@apache.org>.
This should be fixed. I'll keep an eye on Jenkins builds.

Kevin Risden


On Sun, Oct 16, 2022 at 2:18 PM Kevin Risden <kr...@apache.org> wrote:

> This was caused by the Gradle upgrade. I have a fix locally just running
> through tests before pushing.
>
> Kevin Risden
>
>
> On Sat, Oct 15, 2022 at 8:00 PM Kevin Risden <kr...@apache.org> wrote:
>
>> Most likely one of my commits today caused this. Apparently I didn't run
>> the hdfs tests... My guess is commons-text or Gradle. I won't be able to
>> look at this until tomorrow at the earliest.
>>
>> Kevin Risden
>>
>> On Sat, Oct 15, 2022, 19:55 Policeman Jenkins Server <je...@thetaphi.de>
>> wrote:
>>
>>> Build: https://jenkins.thetaphi.de/job/Solr-main-Linux/7849/
>>> Java: 64bit/jdk-16.0.2 -XX:-UseCompressedOops -XX:+UseSerialGC
>>>
>>> 12 tests failed.
>>> FAILED:
>>> org.apache.solr.hdfs.cloud.api.collections.HdfsCloudIncrementalBackupTest.classMethod
>>>
>>> Error Message:
>>> java.lang.Exception: Error starting up MiniSolrCloudCluster
>>>
>>> Stack Trace:
>>> java.lang.Exception: Error starting up MiniSolrCloudCluster
>>>         at __randomizedtesting.SeedInfo.seed([183F02B0C82BE2D2]:0)
>>>         at
>>> org.apache.solr.cloud.MiniSolrCloudCluster.checkForExceptions(MiniSolrCloudCluster.java:767)
>>>         at
>>> org.apache.solr.cloud.MiniSolrCloudCluster.<init>(MiniSolrCloudCluster.java:362)
>>>         at
>>> org.apache.solr.cloud.MiniSolrCloudCluster$Builder.build(MiniSolrCloudCluster.java:1233)
>>>         at
>>> org.apache.solr.cloud.MiniSolrCloudCluster$Builder.configure(MiniSolrCloudCluster.java:1205)
>>>         at
>>> org.apache.solr.hdfs.cloud.api.collections.HdfsCloudIncrementalBackupTest.setupClass(HdfsCloudIncrementalBackupTest.java:129)
>>>         at
>>> java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native
>>> Method)
>>>         at
>>> java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:78)
>>>         at
>>> java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>>>         at java.base/java.lang.reflect.Method.invoke(Method.java:567)
>>>         at
>>> com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
>>>         at
>>> com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:886)
>>>         at
>>> com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
>>>         at
>>> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>>         at
>>> org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
>>>         at
>>> org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
>>>         at
>>> com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
>>>         at org.junit.rules.RunRules.evaluate(RunRules.java:20)
>>>         at
>>> org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
>>>         at
>>> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>>         at
>>> org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
>>>         at
>>> com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
>>>         at
>>> com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
>>>         at
>>> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>>         at
>>> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>>         at
>>> org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
>>>         at
>>> org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
>>>         at
>>> org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
>>>         at
>>> org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
>>>         at
>>> org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
>>>         at org.junit.rules.RunRules.evaluate(RunRules.java:20)
>>>         at
>>> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>>         at
>>> com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
>>>         at
>>> com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
>>>         at java.base/java.lang.Thread.run(Thread.java:831)
>>>         Suppressed: java.lang.NoClassDefFoundError: Could not initialize
>>> class org.apache.hadoop.shaded.com
>>> .sun.xml.bind.v2.runtime.reflect.opt.Injector
>>>                 at org.apache.hadoop.shaded.com
>>> .sun.xml.bind.v2.runtime.reflect.opt.AccessorInjector.prepare(AccessorInjector.java:83)
>>>                 at org.apache.hadoop.shaded.com
>>> .sun.xml.bind.v2.runtime.reflect.opt.OptimizedAccessorFactory.get(OptimizedAccessorFactory.java:176)
>>>                 at org.apache.hadoop.shaded.com
>>> .sun.xml.bind.v2.runtime.reflect.Accessor$FieldReflection.optimize(Accessor.java:282)
>>>                 at org.apache.hadoop.shaded.com
>>> .sun.xml.bind.v2.runtime.property.ArrayProperty.<init>(ArrayProperty.java:69)
>>>                 at org.apache.hadoop.shaded.com
>>> .sun.xml.bind.v2.runtime.property.ArrayERProperty.<init>(ArrayERProperty.java:88)
>>>                 at org.apache.hadoop.shaded.com
>>> .sun.xml.bind.v2.runtime.property.ArrayElementProperty.<init>(ArrayElementProperty.java:100)
>>>                 at org.apache.hadoop.shaded.com
>>> .sun.xml.bind.v2.runtime.property.ArrayElementNodeProperty.<init>(ArrayElementNodeProperty.java:62)
>>>                 at
>>> java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native
>>> Method)
>>>                 at
>>> java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:78)
>>>                 at
>>> java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
>>>                 at
>>> java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499)
>>>                 at
>>> java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480)
>>>                 at org.apache.hadoop.shaded.com
>>> .sun.xml.bind.v2.runtime.property.PropertyFactory.create(PropertyFactory.java:128)
>>>                 at org.apache.hadoop.shaded.com
>>> .sun.xml.bind.v2.runtime.ClassBeanInfoImpl.<init>(ClassBeanInfoImpl.java:183)
>>>                 at org.apache.hadoop.shaded.com
>>> .sun.xml.bind.v2.runtime.JAXBContextImpl.getOrCreate(JAXBContextImpl.java:532)
>>>                 at org.apache.hadoop.shaded.com
>>> .sun.xml.bind.v2.runtime.JAXBContextImpl.<init>(JAXBContextImpl.java:347)
>>>                 at org.apache.hadoop.shaded.com
>>> .sun.xml.bind.v2.runtime.JAXBContextImpl$JAXBContextBuilder.build(JAXBContextImpl.java:1170)
>>>                 at org.apache.hadoop.shaded.com
>>> .sun.xml.bind.v2.ContextFactory.createContext(ContextFactory.java:145)
>>>                 at org.apache.hadoop.shaded.com
>>> .sun.xml.bind.v2.ContextFactory.createContext(ContextFactory.java:236)
>>>                 at
>>> java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native
>>> Method)
>>>                 at
>>> java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:78)
>>>                 at
>>> java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>>>                 at
>>> java.base/java.lang.reflect.Method.invoke(Method.java:567)
>>>                 at
>>> javax.xml.bind.ContextFinder.newInstance(ContextFinder.java:171)
>>>                 at
>>> javax.xml.bind.ContextFinder.newInstance(ContextFinder.java:129)
>>>                 at
>>> javax.xml.bind.ContextFinder.find(ContextFinder.java:307)
>>>                 at
>>> javax.xml.bind.JAXBContext.newInstance(JAXBContext.java:478)
>>>                 at
>>> javax.xml.bind.JAXBContext.newInstance(JAXBContext.java:435)
>>>                 at
>>> org.glassfish.jersey.server.wadl.internal.WadlApplicationContextImpl.getJAXBContextFromWadlGenerator(WadlApplicationContextImpl.java:121)
>>>                 at
>>> org.glassfish.jersey.server.wadl.internal.WadlApplicationContextImpl.isJaxbImplAvailable(WadlApplicationContextImpl.java:270)
>>>                 at
>>> org.glassfish.jersey.server.wadl.WadlFeature.configure(WadlFeature.java:65)
>>>                 at
>>> org.glassfish.jersey.model.internal.CommonConfig.configureFeatures(CommonConfig.java:728)
>>>                 at
>>> org.glassfish.jersey.model.internal.CommonConfig.configureMetaProviders(CommonConfig.java:647)
>>>                 at
>>> org.glassfish.jersey.server.ResourceConfig.configureMetaProviders(ResourceConfig.java:824)
>>>                 at
>>> org.glassfish.jersey.server.ApplicationHandler.initialize(ApplicationHandler.java:332)
>>>                 at
>>> org.glassfish.jersey.server.ApplicationHandler.lambda$initialize$1(ApplicationHandler.java:297)
>>>                 at
>>> org.glassfish.jersey.internal.Errors.process(Errors.java:292)
>>>                 at
>>> org.glassfish.jersey.internal.Errors.process(Errors.java:274)
>>>                 at
>>> org.glassfish.jersey.internal.Errors.processWithException(Errors.java:232)
>>>                 at
>>> org.glassfish.jersey.server.ApplicationHandler.initialize(ApplicationHandler.java:296)
>>>                 at
>>> org.glassfish.jersey.server.ApplicationHandler.<init>(ApplicationHandler.java:261)
>>>                 at
>>> org.glassfish.jersey.server.ApplicationHandler.<init>(ApplicationHandler.java:236)
>>>                 at
>>> org.apache.solr.core.CoreContainer.load(CoreContainer.java:1086)
>>>                 at
>>> org.apache.solr.servlet.CoreContainerProvider.createCoreContainer(CoreContainerProvider.java:377)
>>>                 at
>>> org.apache.solr.servlet.CoreContainerProvider.init(CoreContainerProvider.java:226)
>>>                 at
>>> org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:402)
>>>                 at
>>> org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:194)
>>>                 at
>>> org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:74)
>>>                 at
>>> org.apache.solr.client.solrj.embedded.JettySolrRunner.retryOnPortBindFailure(JettySolrRunner.java:614)
>>>                 at
>>> org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:552)
>>>                 at
>>> org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:523)
>>>                 at
>>> org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:550)
>>>                 at
>>> org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:355)
>>>                 at
>>> java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
>>>                 at
>>> org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:271)
>>>                 at
>>> java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
>>>                 at
>>> java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
>>>                 ... 1 more
>>>         Suppressed: java.lang.ExceptionInInitializerError
>>>                 at org.apache.hadoop.shaded.com
>>> .sun.xml.bind.v2.runtime.reflect.opt.AccessorInjector.prepare(AccessorInjector.java:83)
>>>                 at org.apache.hadoop.shaded.com
>>> .sun.xml.bind.v2.runtime.reflect.opt.OptimizedAccessorFactory.get(OptimizedAccessorFactory.java:176)
>>>                 at org.apache.hadoop.shaded.com
>>> .sun.xml.bind.v2.runtime.reflect.Accessor$FieldReflection.optimize(Accessor.java:282)
>>>                 at org.apache.hadoop.shaded.com
>>> .sun.xml.bind.v2.runtime.property.ArrayProperty.<init>(ArrayProperty.java:69)
>>>                 at org.apache.hadoop.shaded.com
>>> .sun.xml.bind.v2.runtime.property.ArrayERProperty.<init>(ArrayERProperty.java:88)
>>>                 at org.apache.hadoop.shaded.com
>>> .sun.xml.bind.v2.runtime.property.ArrayElementProperty.<init>(ArrayElementProperty.java:100)
>>>                 at org.apache.hadoop.shaded.com
>>> .sun.xml.bind.v2.runtime.property.ArrayElementNodeProperty.<init>(ArrayElementNodeProperty.java:62)
>>>                 at
>>> java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native
>>> Method)
>>>                 at
>>> java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:78)
>>>                 at
>>> java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
>>>                 at
>>> java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499)
>>>                 at
>>> java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480)
>>>                 at org.apache.hadoop.shaded.com
>>> .sun.xml.bind.v2.runtime.property.PropertyFactory.create(PropertyFactory.java:128)
>>>                 at org.apache.hadoop.shaded.com
>>> .sun.xml.bind.v2.runtime.ClassBeanInfoImpl.<init>(ClassBeanInfoImpl.java:183)
>>>                 at org.apache.hadoop.shaded.com
>>> .sun.xml.bind.v2.runtime.JAXBContextImpl.getOrCreate(JAXBContextImpl.java:532)
>>>                 at org.apache.hadoop.shaded.com
>>> .sun.xml.bind.v2.runtime.JAXBContextImpl.<init>(JAXBContextImpl.java:347)
>>>                 at org.apache.hadoop.shaded.com
>>> .sun.xml.bind.v2.runtime.JAXBContextImpl$JAXBContextBuilder.build(JAXBContextImpl.java:1170)
>>>                 at org.apache.hadoop.shaded.com
>>> .sun.xml.bind.v2.ContextFactory.createContext(ContextFactory.java:145)
>>>                 at org.apache.hadoop.shaded.com
>>> .sun.xml.bind.v2.ContextFactory.createContext(ContextFactory.java:236)
>>>                 at
>>> java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native
>>> Method)
>>>                 at
>>> java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:78)
>>>                 at
>>> java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>>>                 at
>>> java.base/java.lang.reflect.Method.invoke(Method.java:567)
>>>                 at
>>> javax.xml.bind.ContextFinder.newInstance(ContextFinder.java:171)
>>>                 at
>>> javax.xml.bind.ContextFinder.newInstance(ContextFinder.java:129)
>>>                 at
>>> javax.xml.bind.ContextFinder.find(ContextFinder.java:307)
>>>                 at
>>> javax.xml.bind.JAXBContext.newInstance(JAXBContext.java:478)
>>>                 at
>>> javax.xml.bind.JAXBContext.newInstance(JAXBContext.java:435)
>>>                 at
>>> org.glassfish.jersey.server.wadl.internal.WadlApplicationContextImpl.getJAXBContextFromWadlGenerator(WadlApplicationContextImpl.java:121)
>>>                 at
>>> org.glassfish.jersey.server.wadl.internal.WadlApplicationContextImpl.isJaxbImplAvailable(WadlApplicationContextImpl.java:270)
>>>                 at
>>> org.glassfish.jersey.server.wadl.WadlFeature.configure(WadlFeature.java:65)
>>>                 at
>>> org.glassfish.jersey.model.internal.CommonConfig.configureFeatures(CommonConfig.java:728)
>>>                 at
>>> org.glassfish.jersey.model.internal.CommonConfig.configureMetaProviders(CommonConfig.java:647)
>>>                 at
>>> org.glassfish.jersey.server.ResourceConfig.configureMetaProviders(ResourceConfig.java:824)
>>>                 at
>>> org.glassfish.jersey.server.ApplicationHandler.initialize(ApplicationHandler.java:332)
>>>                 at
>>> org.glassfish.jersey.server.ApplicationHandler.lambda$initialize$1(ApplicationHandler.java:297)
>>>                 at
>>> org.glassfish.jersey.internal.Errors.process(Errors.java:292)
>>>                 at
>>> org.glassfish.jersey.internal.Errors.process(Errors.java:274)
>>>                 at
>>> org.glassfish.jersey.internal.Errors.processWithException(Errors.java:232)
>>>                 at
>>> org.glassfish.jersey.server.ApplicationHandler.initialize(ApplicationHandler.java:296)
>>>                 at
>>> org.glassfish.jersey.server.ApplicationHandler.<init>(ApplicationHandler.java:261)
>>>                 at
>>> org.glassfish.jersey.server.ApplicationHandler.<init>(ApplicationHandler.java:236)
>>>                 at
>>> org.apache.solr.core.CoreContainer.load(CoreContainer.java:1086)
>>>                 at
>>> org.apache.solr.servlet.CoreContainerProvider.createCoreContainer(CoreContainerProvider.java:377)
>>>                 at
>>> org.apache.solr.servlet.CoreContainerProvider.init(CoreContainerProvider.java:226)
>>>                 at
>>> org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:402)
>>>                 at
>>> org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:194)
>>>                 at
>>> org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:74)
>>>                 at
>>> org.apache.solr.client.solrj.embedded.JettySolrRunner.retryOnPortBindFailure(JettySolrRunner.java:614)
>>>                 at
>>> org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:552)
>>>                 at
>>> org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:523)
>>>                 at
>>> org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:550)
>>>                 at
>>> org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:355)
>>>                 at
>>> java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
>>>                 at
>>> org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:271)
>>>                 at
>>> java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
>>>                 at
>>> java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
>>>                 ... 1 more
>>>         Caused by: java.lang.reflect.InaccessibleObjectException: Unable
>>> to make protected final java.lang.Class
>>> java.lang.ClassLoader.defineClass(java.lang.String,byte[],int,int) throws
>>> java.lang.ClassFormatError accessible: module java.base does not "opens
>>> java.lang" to unnamed module @77f1baf5
>>>                 at
>>> java.base/java.lang.reflect.AccessibleObject.checkCanSetAccessible(AccessibleObject.java:357)
>>>                 at
>>> java.base/java.lang.reflect.AccessibleObject.checkCanSetAccessible(AccessibleObject.java:297)
>>>                 at
>>> java.base/java.lang.reflect.Method.checkCanSetAccessible(Method.java:199)
>>>                 at
>>> java.base/java.lang.reflect.Method.setAccessible(Method.java:193)
>>>                 at org.apache.hadoop.shaded.com
>>> .sun.xml.bind.v2.runtime.reflect.opt.Injector$1.run(Injector.java:177)
>>>                 at org.apache.hadoop.shaded.com
>>> .sun.xml.bind.v2.runtime.reflect.opt.Injector$1.run(Injector.java:174)
>>>                 at
>>> java.base/java.security.AccessController.doPrivileged(AccessController.java:312)
>>>                 at org.apache.hadoop.shaded.com
>>> .sun.xml.bind.v2.runtime.reflect.opt.Injector.<clinit>(Injector.java:172)
>>>                 ... 58 more
>>>
>>>
>>> FAILED:
>>> org.apache.solr.hdfs.cloud.api.collections.HdfsCloudIncrementalBackupTest.classMethod
>>>
>>> Error Message:
>>> com.carrotsearch.randomizedtesting.ThreadLeakError: 46 threads leaked
>>> from SUITE scope at
>>> org.apache.solr.hdfs.cloud.api.collections.HdfsCloudIncrementalBackupTest:
>>>    1) Thread[id=401, name=CloserThread, state=WAITING,
>>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>>         at java.base@16.0.2/java.lang.Object.wait(Native Method)
>>>         at java.base@16.0.2/java.lang.Object.wait(Object.java:320)
>>>         at
>>> app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2506)
>>>    2) Thread[id=372, name=Session-HouseKeeper-74f5e0-1,
>>> state=TIMED_WAITING, group=TGRP-HdfsCloudIncrementalBackupTest]
>>>         at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
>>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>>    3) Thread[id=400,
>>> name=OverseerCollectionConfigSetProcessor-72262794004201478-127.0.0.1:46753_solr-n_0000000000,
>>> state=WAITING, group=Overseer collection creation process.]
>>>         at java.base@16.0.2/java.lang.Object.wait(Native Method)
>>>         at java.base@16.0.2/java.lang.Object.wait(Object.java:320)
>>>         at
>>> app//org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1590)
>>>         at
>>> app//org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1562)
>>>         at
>>> app//org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1970)
>>>         at
>>> app//org.apache.solr.common.cloud.SolrZkClient.lambda$getData$6(SolrZkClient.java:432)
>>>         at
>>> app//org.apache.solr.common.cloud.SolrZkClient$$Lambda$382/0x00000008013d7858.execute(Unknown
>>> Source)
>>>         at
>>> app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:72)
>>>         at
>>> app//org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:432)
>>>         at
>>> app//org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:486)
>>>         at
>>> app//org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:232)
>>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>>    4) Thread[id=367, name=qtp1790869424-367, state=RUNNABLE,
>>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .SelectorImpl.select(SelectorImpl.java:146)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector.select(ManagedSelector.java:190)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>>>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
>>> Source)
>>>         at
>>> app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
>>>         at
>>> app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
>>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>>    5) Thread[id=360, name=qtp577348919-360, state=RUNNABLE,
>>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .SelectorImpl.select(SelectorImpl.java:146)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector.select(ManagedSelector.java:190)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>>>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
>>> Source)
>>>         at
>>> app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
>>>         at
>>> app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
>>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>>    6) Thread[id=404, name=coreContainerWorkExecutor-18-thread-1,
>>> state=TIMED_WAITING, group=TGRP-HdfsCloudIncrementalBackupTest]
>>>         at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
>>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>>    7) Thread[id=383, name=httpShardExecutor-21-thread-1, state=RUNNABLE,
>>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .SelectorImpl.select(SelectorImpl.java:146)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector.select(ManagedSelector.java:190)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>>>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
>>> Source)
>>>         at
>>> app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:271)
>>>         at
>>> app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$349/0x0000000801384860.run(Unknown
>>> Source)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
>>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>>    8) Thread[id=364, name=qtp1790869424-364, state=RUNNABLE,
>>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .SelectorImpl.select(SelectorImpl.java:146)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector.select(ManagedSelector.java:190)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>>>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
>>> Source)
>>>         at
>>> app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
>>>         at
>>> app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
>>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>>    9) Thread[id=428, name=zkCallback-32-thread-4, state=TIMED_WAITING,
>>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>>         at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
>>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>>   10) Thread[id=418, name=zkCallback-32-thread-2, state=TIMED_WAITING,
>>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>>         at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
>>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>>   11) Thread[id=365, name=qtp577348919-365, state=RUNNABLE,
>>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .SelectorImpl.select(SelectorImpl.java:146)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector.select(ManagedSelector.java:190)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>>>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
>>> Source)
>>>         at
>>> app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
>>>         at
>>> app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
>>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>>   12) Thread[id=369, name=qtp1790869424-369, state=RUNNABLE,
>>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .SelectorImpl.select(SelectorImpl.java:146)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector.select(ManagedSelector.java:190)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>>>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
>>> Source)
>>>         at
>>> app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
>>>         at
>>> app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
>>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>>   13) Thread[id=387, name=httpShardExecutor-26-thread-1, state=RUNNABLE,
>>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .SelectorImpl.select(SelectorImpl.java:146)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector.select(ManagedSelector.java:190)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>>>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
>>> Source)
>>>         at
>>> app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:271)
>>>         at
>>> app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$349/0x0000000801384860.run(Unknown
>>> Source)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
>>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>>   14) Thread[id=355, name=qtp577348919-355, state=RUNNABLE,
>>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .SelectorImpl.select(SelectorImpl.java:146)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector.select(ManagedSelector.java:190)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>>>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
>>> Source)
>>>         at
>>> app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
>>>         at
>>> app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
>>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>>   15) Thread[id=396, name=zkConnectionManagerCallback-33-thread-1,
>>> state=WAITING, group=TGRP-HdfsCloudIncrementalBackupTest]
>>>         at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
>>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>>   16) Thread[id=362, name=qtp577348919-362, state=RUNNABLE,
>>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .SelectorImpl.select(SelectorImpl.java:146)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector.select(ManagedSelector.java:190)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>>>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
>>> Source)
>>>         at
>>> app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
>>>         at
>>> app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
>>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>>   17) Thread[id=363, name=qtp577348919-363, state=RUNNABLE,
>>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .SelectorImpl.select(SelectorImpl.java:146)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector.select(ManagedSelector.java:190)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>>>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
>>> Source)
>>>         at
>>> app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
>>>         at
>>> app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
>>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>>   18) Thread[id=389, name=jetty-launcher-9-thread-2-EventThread,
>>> state=WAITING, group=TGRP-HdfsCloudIncrementalBackupTest]
>>>         at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
>>>         at
>>> app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:549)
>>>   19) Thread[id=374,
>>> name=qtp1790869424-374-acceptor-0@31e11e51-ServerConnector@442851d5{ssl,
>>> (ssl, alpn, http/1.1, h2)}{127.0.0.1:44953}, state=RUNNABLE,
>>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>>         at java.base@16.0.2/sun.nio.ch.Net.accept(Native Method)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .ServerSocketChannelImpl.implAccept(ServerSocketChannelImpl.java:424)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:390)
>>>         at
>>> app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388)
>>>         at
>>> app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:704)
>>>         at
>>> app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
>>>         at
>>> app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
>>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>>   20) Thread[id=356, name=qtp1790869424-356, state=RUNNABLE,
>>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .SelectorImpl.select(SelectorImpl.java:146)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector.select(ManagedSelector.java:190)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>>>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
>>> Source)
>>>         at
>>> app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
>>>         at
>>> app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
>>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>>   21) Thread[id=417, name=zkCallback-27-thread-2, state=TIMED_WAITING,
>>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>>         at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
>>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>>   22) Thread[id=359, name=qtp577348919-359, state=RUNNABLE,
>>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .SelectorImpl.select(SelectorImpl.java:146)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector.select(ManagedSelector.java:190)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>>>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
>>> Source)
>>>         at
>>> app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
>>>         at
>>> app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
>>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>>   23) Thread[id=386, name=h2sc-23-thread-1, state=RUNNABLE,
>>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .SelectorImpl.select(SelectorImpl.java:146)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector.select(ManagedSelector.java:190)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>>>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
>>> Source)
>>>         at
>>> app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:271)
>>>         at
>>> app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$349/0x0000000801384860.run(Unknown
>>> Source)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
>>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>>   24) Thread[id=366, name=qtp1790869424-366, state=RUNNABLE,
>>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .SelectorImpl.select(SelectorImpl.java:146)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector.select(ManagedSelector.java:190)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>>>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
>>> Source)
>>>         at
>>> app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
>>>         at
>>> app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
>>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>>   25) Thread[id=402, name=CloserThread, state=WAITING,
>>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>>         at java.base@16.0.2/java.lang.Object.wait(Native Method)
>>>         at java.base@16.0.2/java.lang.Object.wait(Object.java:320)
>>>         at
>>> app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2506)
>>>   26) Thread[id=361, name=qtp1790869424-361, state=RUNNABLE,
>>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .SelectorImpl.select(SelectorImpl.java:146)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector.select(ManagedSelector.java:190)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>>>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
>>> Source)
>>>         at
>>> app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
>>>         at
>>> app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
>>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>>   27) Thread[id=393, name=h2sc-30-thread-1, state=RUNNABLE,
>>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>>>
>>

Re: [JENKINS] Solr-main-Linux (64bit/jdk-16.0.2) - Build # 7849 - Still Unstable!

Posted by Kevin Risden <kr...@apache.org>.
This should be fixed. I'll keep an eye on Jenkins builds.

Kevin Risden


On Sun, Oct 16, 2022 at 2:18 PM Kevin Risden <kr...@apache.org> wrote:

> This was caused by the Gradle upgrade. I have a fix locally just running
> through tests before pushing.
>
> Kevin Risden
>
>
> On Sat, Oct 15, 2022 at 8:00 PM Kevin Risden <kr...@apache.org> wrote:
>
>> Most likely one of my commits today caused this. Apparently I didn't run
>> the hdfs tests... My guess is commons-text or Gradle. I won't be able to
>> look at this until tomorrow at the earliest.
>>
>> Kevin Risden
>>
>> On Sat, Oct 15, 2022, 19:55 Policeman Jenkins Server <je...@thetaphi.de>
>> wrote:
>>
>>> Build: https://jenkins.thetaphi.de/job/Solr-main-Linux/7849/
>>> Java: 64bit/jdk-16.0.2 -XX:-UseCompressedOops -XX:+UseSerialGC
>>>
>>> 12 tests failed.
>>> FAILED:
>>> org.apache.solr.hdfs.cloud.api.collections.HdfsCloudIncrementalBackupTest.classMethod
>>>
>>> Error Message:
>>> java.lang.Exception: Error starting up MiniSolrCloudCluster
>>>
>>> Stack Trace:
>>> java.lang.Exception: Error starting up MiniSolrCloudCluster
>>>         at __randomizedtesting.SeedInfo.seed([183F02B0C82BE2D2]:0)
>>>         at
>>> org.apache.solr.cloud.MiniSolrCloudCluster.checkForExceptions(MiniSolrCloudCluster.java:767)
>>>         at
>>> org.apache.solr.cloud.MiniSolrCloudCluster.<init>(MiniSolrCloudCluster.java:362)
>>>         at
>>> org.apache.solr.cloud.MiniSolrCloudCluster$Builder.build(MiniSolrCloudCluster.java:1233)
>>>         at
>>> org.apache.solr.cloud.MiniSolrCloudCluster$Builder.configure(MiniSolrCloudCluster.java:1205)
>>>         at
>>> org.apache.solr.hdfs.cloud.api.collections.HdfsCloudIncrementalBackupTest.setupClass(HdfsCloudIncrementalBackupTest.java:129)
>>>         at
>>> java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native
>>> Method)
>>>         at
>>> java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:78)
>>>         at
>>> java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>>>         at java.base/java.lang.reflect.Method.invoke(Method.java:567)
>>>         at
>>> com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
>>>         at
>>> com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:886)
>>>         at
>>> com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
>>>         at
>>> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>>         at
>>> org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
>>>         at
>>> org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
>>>         at
>>> com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
>>>         at org.junit.rules.RunRules.evaluate(RunRules.java:20)
>>>         at
>>> org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
>>>         at
>>> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>>         at
>>> org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
>>>         at
>>> com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
>>>         at
>>> com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
>>>         at
>>> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>>         at
>>> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>>         at
>>> org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
>>>         at
>>> org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
>>>         at
>>> org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
>>>         at
>>> org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
>>>         at
>>> org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
>>>         at org.junit.rules.RunRules.evaluate(RunRules.java:20)
>>>         at
>>> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>>         at
>>> com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
>>>         at
>>> com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
>>>         at java.base/java.lang.Thread.run(Thread.java:831)
>>>         Suppressed: java.lang.NoClassDefFoundError: Could not initialize
>>> class org.apache.hadoop.shaded.com
>>> .sun.xml.bind.v2.runtime.reflect.opt.Injector
>>>                 at org.apache.hadoop.shaded.com
>>> .sun.xml.bind.v2.runtime.reflect.opt.AccessorInjector.prepare(AccessorInjector.java:83)
>>>                 at org.apache.hadoop.shaded.com
>>> .sun.xml.bind.v2.runtime.reflect.opt.OptimizedAccessorFactory.get(OptimizedAccessorFactory.java:176)
>>>                 at org.apache.hadoop.shaded.com
>>> .sun.xml.bind.v2.runtime.reflect.Accessor$FieldReflection.optimize(Accessor.java:282)
>>>                 at org.apache.hadoop.shaded.com
>>> .sun.xml.bind.v2.runtime.property.ArrayProperty.<init>(ArrayProperty.java:69)
>>>                 at org.apache.hadoop.shaded.com
>>> .sun.xml.bind.v2.runtime.property.ArrayERProperty.<init>(ArrayERProperty.java:88)
>>>                 at org.apache.hadoop.shaded.com
>>> .sun.xml.bind.v2.runtime.property.ArrayElementProperty.<init>(ArrayElementProperty.java:100)
>>>                 at org.apache.hadoop.shaded.com
>>> .sun.xml.bind.v2.runtime.property.ArrayElementNodeProperty.<init>(ArrayElementNodeProperty.java:62)
>>>                 at
>>> java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native
>>> Method)
>>>                 at
>>> java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:78)
>>>                 at
>>> java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
>>>                 at
>>> java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499)
>>>                 at
>>> java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480)
>>>                 at org.apache.hadoop.shaded.com
>>> .sun.xml.bind.v2.runtime.property.PropertyFactory.create(PropertyFactory.java:128)
>>>                 at org.apache.hadoop.shaded.com
>>> .sun.xml.bind.v2.runtime.ClassBeanInfoImpl.<init>(ClassBeanInfoImpl.java:183)
>>>                 at org.apache.hadoop.shaded.com
>>> .sun.xml.bind.v2.runtime.JAXBContextImpl.getOrCreate(JAXBContextImpl.java:532)
>>>                 at org.apache.hadoop.shaded.com
>>> .sun.xml.bind.v2.runtime.JAXBContextImpl.<init>(JAXBContextImpl.java:347)
>>>                 at org.apache.hadoop.shaded.com
>>> .sun.xml.bind.v2.runtime.JAXBContextImpl$JAXBContextBuilder.build(JAXBContextImpl.java:1170)
>>>                 at org.apache.hadoop.shaded.com
>>> .sun.xml.bind.v2.ContextFactory.createContext(ContextFactory.java:145)
>>>                 at org.apache.hadoop.shaded.com
>>> .sun.xml.bind.v2.ContextFactory.createContext(ContextFactory.java:236)
>>>                 at
>>> java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native
>>> Method)
>>>                 at
>>> java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:78)
>>>                 at
>>> java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>>>                 at
>>> java.base/java.lang.reflect.Method.invoke(Method.java:567)
>>>                 at
>>> javax.xml.bind.ContextFinder.newInstance(ContextFinder.java:171)
>>>                 at
>>> javax.xml.bind.ContextFinder.newInstance(ContextFinder.java:129)
>>>                 at
>>> javax.xml.bind.ContextFinder.find(ContextFinder.java:307)
>>>                 at
>>> javax.xml.bind.JAXBContext.newInstance(JAXBContext.java:478)
>>>                 at
>>> javax.xml.bind.JAXBContext.newInstance(JAXBContext.java:435)
>>>                 at
>>> org.glassfish.jersey.server.wadl.internal.WadlApplicationContextImpl.getJAXBContextFromWadlGenerator(WadlApplicationContextImpl.java:121)
>>>                 at
>>> org.glassfish.jersey.server.wadl.internal.WadlApplicationContextImpl.isJaxbImplAvailable(WadlApplicationContextImpl.java:270)
>>>                 at
>>> org.glassfish.jersey.server.wadl.WadlFeature.configure(WadlFeature.java:65)
>>>                 at
>>> org.glassfish.jersey.model.internal.CommonConfig.configureFeatures(CommonConfig.java:728)
>>>                 at
>>> org.glassfish.jersey.model.internal.CommonConfig.configureMetaProviders(CommonConfig.java:647)
>>>                 at
>>> org.glassfish.jersey.server.ResourceConfig.configureMetaProviders(ResourceConfig.java:824)
>>>                 at
>>> org.glassfish.jersey.server.ApplicationHandler.initialize(ApplicationHandler.java:332)
>>>                 at
>>> org.glassfish.jersey.server.ApplicationHandler.lambda$initialize$1(ApplicationHandler.java:297)
>>>                 at
>>> org.glassfish.jersey.internal.Errors.process(Errors.java:292)
>>>                 at
>>> org.glassfish.jersey.internal.Errors.process(Errors.java:274)
>>>                 at
>>> org.glassfish.jersey.internal.Errors.processWithException(Errors.java:232)
>>>                 at
>>> org.glassfish.jersey.server.ApplicationHandler.initialize(ApplicationHandler.java:296)
>>>                 at
>>> org.glassfish.jersey.server.ApplicationHandler.<init>(ApplicationHandler.java:261)
>>>                 at
>>> org.glassfish.jersey.server.ApplicationHandler.<init>(ApplicationHandler.java:236)
>>>                 at
>>> org.apache.solr.core.CoreContainer.load(CoreContainer.java:1086)
>>>                 at
>>> org.apache.solr.servlet.CoreContainerProvider.createCoreContainer(CoreContainerProvider.java:377)
>>>                 at
>>> org.apache.solr.servlet.CoreContainerProvider.init(CoreContainerProvider.java:226)
>>>                 at
>>> org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:402)
>>>                 at
>>> org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:194)
>>>                 at
>>> org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:74)
>>>                 at
>>> org.apache.solr.client.solrj.embedded.JettySolrRunner.retryOnPortBindFailure(JettySolrRunner.java:614)
>>>                 at
>>> org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:552)
>>>                 at
>>> org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:523)
>>>                 at
>>> org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:550)
>>>                 at
>>> org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:355)
>>>                 at
>>> java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
>>>                 at
>>> org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:271)
>>>                 at
>>> java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
>>>                 at
>>> java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
>>>                 ... 1 more
>>>         Suppressed: java.lang.ExceptionInInitializerError
>>>                 at org.apache.hadoop.shaded.com
>>> .sun.xml.bind.v2.runtime.reflect.opt.AccessorInjector.prepare(AccessorInjector.java:83)
>>>                 at org.apache.hadoop.shaded.com
>>> .sun.xml.bind.v2.runtime.reflect.opt.OptimizedAccessorFactory.get(OptimizedAccessorFactory.java:176)
>>>                 at org.apache.hadoop.shaded.com
>>> .sun.xml.bind.v2.runtime.reflect.Accessor$FieldReflection.optimize(Accessor.java:282)
>>>                 at org.apache.hadoop.shaded.com
>>> .sun.xml.bind.v2.runtime.property.ArrayProperty.<init>(ArrayProperty.java:69)
>>>                 at org.apache.hadoop.shaded.com
>>> .sun.xml.bind.v2.runtime.property.ArrayERProperty.<init>(ArrayERProperty.java:88)
>>>                 at org.apache.hadoop.shaded.com
>>> .sun.xml.bind.v2.runtime.property.ArrayElementProperty.<init>(ArrayElementProperty.java:100)
>>>                 at org.apache.hadoop.shaded.com
>>> .sun.xml.bind.v2.runtime.property.ArrayElementNodeProperty.<init>(ArrayElementNodeProperty.java:62)
>>>                 at
>>> java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native
>>> Method)
>>>                 at
>>> java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:78)
>>>                 at
>>> java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
>>>                 at
>>> java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499)
>>>                 at
>>> java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480)
>>>                 at org.apache.hadoop.shaded.com
>>> .sun.xml.bind.v2.runtime.property.PropertyFactory.create(PropertyFactory.java:128)
>>>                 at org.apache.hadoop.shaded.com
>>> .sun.xml.bind.v2.runtime.ClassBeanInfoImpl.<init>(ClassBeanInfoImpl.java:183)
>>>                 at org.apache.hadoop.shaded.com
>>> .sun.xml.bind.v2.runtime.JAXBContextImpl.getOrCreate(JAXBContextImpl.java:532)
>>>                 at org.apache.hadoop.shaded.com
>>> .sun.xml.bind.v2.runtime.JAXBContextImpl.<init>(JAXBContextImpl.java:347)
>>>                 at org.apache.hadoop.shaded.com
>>> .sun.xml.bind.v2.runtime.JAXBContextImpl$JAXBContextBuilder.build(JAXBContextImpl.java:1170)
>>>                 at org.apache.hadoop.shaded.com
>>> .sun.xml.bind.v2.ContextFactory.createContext(ContextFactory.java:145)
>>>                 at org.apache.hadoop.shaded.com
>>> .sun.xml.bind.v2.ContextFactory.createContext(ContextFactory.java:236)
>>>                 at
>>> java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native
>>> Method)
>>>                 at
>>> java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:78)
>>>                 at
>>> java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>>>                 at
>>> java.base/java.lang.reflect.Method.invoke(Method.java:567)
>>>                 at
>>> javax.xml.bind.ContextFinder.newInstance(ContextFinder.java:171)
>>>                 at
>>> javax.xml.bind.ContextFinder.newInstance(ContextFinder.java:129)
>>>                 at
>>> javax.xml.bind.ContextFinder.find(ContextFinder.java:307)
>>>                 at
>>> javax.xml.bind.JAXBContext.newInstance(JAXBContext.java:478)
>>>                 at
>>> javax.xml.bind.JAXBContext.newInstance(JAXBContext.java:435)
>>>                 at
>>> org.glassfish.jersey.server.wadl.internal.WadlApplicationContextImpl.getJAXBContextFromWadlGenerator(WadlApplicationContextImpl.java:121)
>>>                 at
>>> org.glassfish.jersey.server.wadl.internal.WadlApplicationContextImpl.isJaxbImplAvailable(WadlApplicationContextImpl.java:270)
>>>                 at
>>> org.glassfish.jersey.server.wadl.WadlFeature.configure(WadlFeature.java:65)
>>>                 at
>>> org.glassfish.jersey.model.internal.CommonConfig.configureFeatures(CommonConfig.java:728)
>>>                 at
>>> org.glassfish.jersey.model.internal.CommonConfig.configureMetaProviders(CommonConfig.java:647)
>>>                 at
>>> org.glassfish.jersey.server.ResourceConfig.configureMetaProviders(ResourceConfig.java:824)
>>>                 at
>>> org.glassfish.jersey.server.ApplicationHandler.initialize(ApplicationHandler.java:332)
>>>                 at
>>> org.glassfish.jersey.server.ApplicationHandler.lambda$initialize$1(ApplicationHandler.java:297)
>>>                 at
>>> org.glassfish.jersey.internal.Errors.process(Errors.java:292)
>>>                 at
>>> org.glassfish.jersey.internal.Errors.process(Errors.java:274)
>>>                 at
>>> org.glassfish.jersey.internal.Errors.processWithException(Errors.java:232)
>>>                 at
>>> org.glassfish.jersey.server.ApplicationHandler.initialize(ApplicationHandler.java:296)
>>>                 at
>>> org.glassfish.jersey.server.ApplicationHandler.<init>(ApplicationHandler.java:261)
>>>                 at
>>> org.glassfish.jersey.server.ApplicationHandler.<init>(ApplicationHandler.java:236)
>>>                 at
>>> org.apache.solr.core.CoreContainer.load(CoreContainer.java:1086)
>>>                 at
>>> org.apache.solr.servlet.CoreContainerProvider.createCoreContainer(CoreContainerProvider.java:377)
>>>                 at
>>> org.apache.solr.servlet.CoreContainerProvider.init(CoreContainerProvider.java:226)
>>>                 at
>>> org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:402)
>>>                 at
>>> org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:194)
>>>                 at
>>> org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:74)
>>>                 at
>>> org.apache.solr.client.solrj.embedded.JettySolrRunner.retryOnPortBindFailure(JettySolrRunner.java:614)
>>>                 at
>>> org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:552)
>>>                 at
>>> org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:523)
>>>                 at
>>> org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:550)
>>>                 at
>>> org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:355)
>>>                 at
>>> java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
>>>                 at
>>> org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:271)
>>>                 at
>>> java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
>>>                 at
>>> java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
>>>                 ... 1 more
>>>         Caused by: java.lang.reflect.InaccessibleObjectException: Unable
>>> to make protected final java.lang.Class
>>> java.lang.ClassLoader.defineClass(java.lang.String,byte[],int,int) throws
>>> java.lang.ClassFormatError accessible: module java.base does not "opens
>>> java.lang" to unnamed module @77f1baf5
>>>                 at
>>> java.base/java.lang.reflect.AccessibleObject.checkCanSetAccessible(AccessibleObject.java:357)
>>>                 at
>>> java.base/java.lang.reflect.AccessibleObject.checkCanSetAccessible(AccessibleObject.java:297)
>>>                 at
>>> java.base/java.lang.reflect.Method.checkCanSetAccessible(Method.java:199)
>>>                 at
>>> java.base/java.lang.reflect.Method.setAccessible(Method.java:193)
>>>                 at org.apache.hadoop.shaded.com
>>> .sun.xml.bind.v2.runtime.reflect.opt.Injector$1.run(Injector.java:177)
>>>                 at org.apache.hadoop.shaded.com
>>> .sun.xml.bind.v2.runtime.reflect.opt.Injector$1.run(Injector.java:174)
>>>                 at
>>> java.base/java.security.AccessController.doPrivileged(AccessController.java:312)
>>>                 at org.apache.hadoop.shaded.com
>>> .sun.xml.bind.v2.runtime.reflect.opt.Injector.<clinit>(Injector.java:172)
>>>                 ... 58 more
>>>
>>>
>>> FAILED:
>>> org.apache.solr.hdfs.cloud.api.collections.HdfsCloudIncrementalBackupTest.classMethod
>>>
>>> Error Message:
>>> com.carrotsearch.randomizedtesting.ThreadLeakError: 46 threads leaked
>>> from SUITE scope at
>>> org.apache.solr.hdfs.cloud.api.collections.HdfsCloudIncrementalBackupTest:
>>>    1) Thread[id=401, name=CloserThread, state=WAITING,
>>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>>         at java.base@16.0.2/java.lang.Object.wait(Native Method)
>>>         at java.base@16.0.2/java.lang.Object.wait(Object.java:320)
>>>         at
>>> app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2506)
>>>    2) Thread[id=372, name=Session-HouseKeeper-74f5e0-1,
>>> state=TIMED_WAITING, group=TGRP-HdfsCloudIncrementalBackupTest]
>>>         at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
>>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>>    3) Thread[id=400,
>>> name=OverseerCollectionConfigSetProcessor-72262794004201478-127.0.0.1:46753_solr-n_0000000000,
>>> state=WAITING, group=Overseer collection creation process.]
>>>         at java.base@16.0.2/java.lang.Object.wait(Native Method)
>>>         at java.base@16.0.2/java.lang.Object.wait(Object.java:320)
>>>         at
>>> app//org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1590)
>>>         at
>>> app//org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1562)
>>>         at
>>> app//org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1970)
>>>         at
>>> app//org.apache.solr.common.cloud.SolrZkClient.lambda$getData$6(SolrZkClient.java:432)
>>>         at
>>> app//org.apache.solr.common.cloud.SolrZkClient$$Lambda$382/0x00000008013d7858.execute(Unknown
>>> Source)
>>>         at
>>> app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:72)
>>>         at
>>> app//org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:432)
>>>         at
>>> app//org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:486)
>>>         at
>>> app//org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:232)
>>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>>    4) Thread[id=367, name=qtp1790869424-367, state=RUNNABLE,
>>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .SelectorImpl.select(SelectorImpl.java:146)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector.select(ManagedSelector.java:190)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>>>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
>>> Source)
>>>         at
>>> app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
>>>         at
>>> app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
>>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>>    5) Thread[id=360, name=qtp577348919-360, state=RUNNABLE,
>>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .SelectorImpl.select(SelectorImpl.java:146)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector.select(ManagedSelector.java:190)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>>>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
>>> Source)
>>>         at
>>> app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
>>>         at
>>> app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
>>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>>    6) Thread[id=404, name=coreContainerWorkExecutor-18-thread-1,
>>> state=TIMED_WAITING, group=TGRP-HdfsCloudIncrementalBackupTest]
>>>         at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
>>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>>    7) Thread[id=383, name=httpShardExecutor-21-thread-1, state=RUNNABLE,
>>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .SelectorImpl.select(SelectorImpl.java:146)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector.select(ManagedSelector.java:190)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>>>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
>>> Source)
>>>         at
>>> app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:271)
>>>         at
>>> app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$349/0x0000000801384860.run(Unknown
>>> Source)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
>>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>>    8) Thread[id=364, name=qtp1790869424-364, state=RUNNABLE,
>>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .SelectorImpl.select(SelectorImpl.java:146)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector.select(ManagedSelector.java:190)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>>>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
>>> Source)
>>>         at
>>> app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
>>>         at
>>> app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
>>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>>    9) Thread[id=428, name=zkCallback-32-thread-4, state=TIMED_WAITING,
>>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>>         at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
>>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>>   10) Thread[id=418, name=zkCallback-32-thread-2, state=TIMED_WAITING,
>>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>>         at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
>>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>>   11) Thread[id=365, name=qtp577348919-365, state=RUNNABLE,
>>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .SelectorImpl.select(SelectorImpl.java:146)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector.select(ManagedSelector.java:190)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>>>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
>>> Source)
>>>         at
>>> app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
>>>         at
>>> app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
>>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>>   12) Thread[id=369, name=qtp1790869424-369, state=RUNNABLE,
>>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .SelectorImpl.select(SelectorImpl.java:146)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector.select(ManagedSelector.java:190)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>>>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
>>> Source)
>>>         at
>>> app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
>>>         at
>>> app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
>>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>>   13) Thread[id=387, name=httpShardExecutor-26-thread-1, state=RUNNABLE,
>>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .SelectorImpl.select(SelectorImpl.java:146)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector.select(ManagedSelector.java:190)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>>>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
>>> Source)
>>>         at
>>> app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:271)
>>>         at
>>> app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$349/0x0000000801384860.run(Unknown
>>> Source)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
>>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>>   14) Thread[id=355, name=qtp577348919-355, state=RUNNABLE,
>>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .SelectorImpl.select(SelectorImpl.java:146)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector.select(ManagedSelector.java:190)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>>>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
>>> Source)
>>>         at
>>> app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
>>>         at
>>> app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
>>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>>   15) Thread[id=396, name=zkConnectionManagerCallback-33-thread-1,
>>> state=WAITING, group=TGRP-HdfsCloudIncrementalBackupTest]
>>>         at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
>>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>>   16) Thread[id=362, name=qtp577348919-362, state=RUNNABLE,
>>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .SelectorImpl.select(SelectorImpl.java:146)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector.select(ManagedSelector.java:190)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>>>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
>>> Source)
>>>         at
>>> app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
>>>         at
>>> app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
>>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>>   17) Thread[id=363, name=qtp577348919-363, state=RUNNABLE,
>>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .SelectorImpl.select(SelectorImpl.java:146)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector.select(ManagedSelector.java:190)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>>>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
>>> Source)
>>>         at
>>> app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
>>>         at
>>> app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
>>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>>   18) Thread[id=389, name=jetty-launcher-9-thread-2-EventThread,
>>> state=WAITING, group=TGRP-HdfsCloudIncrementalBackupTest]
>>>         at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
>>>         at
>>> app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:549)
>>>   19) Thread[id=374,
>>> name=qtp1790869424-374-acceptor-0@31e11e51-ServerConnector@442851d5{ssl,
>>> (ssl, alpn, http/1.1, h2)}{127.0.0.1:44953}, state=RUNNABLE,
>>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>>         at java.base@16.0.2/sun.nio.ch.Net.accept(Native Method)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .ServerSocketChannelImpl.implAccept(ServerSocketChannelImpl.java:424)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:390)
>>>         at
>>> app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388)
>>>         at
>>> app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:704)
>>>         at
>>> app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
>>>         at
>>> app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
>>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>>   20) Thread[id=356, name=qtp1790869424-356, state=RUNNABLE,
>>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .SelectorImpl.select(SelectorImpl.java:146)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector.select(ManagedSelector.java:190)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>>>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
>>> Source)
>>>         at
>>> app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
>>>         at
>>> app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
>>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>>   21) Thread[id=417, name=zkCallback-27-thread-2, state=TIMED_WAITING,
>>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>>         at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
>>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>>   22) Thread[id=359, name=qtp577348919-359, state=RUNNABLE,
>>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .SelectorImpl.select(SelectorImpl.java:146)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector.select(ManagedSelector.java:190)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>>>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
>>> Source)
>>>         at
>>> app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
>>>         at
>>> app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
>>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>>   23) Thread[id=386, name=h2sc-23-thread-1, state=RUNNABLE,
>>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .SelectorImpl.select(SelectorImpl.java:146)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector.select(ManagedSelector.java:190)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>>>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
>>> Source)
>>>         at
>>> app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:271)
>>>         at
>>> app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$349/0x0000000801384860.run(Unknown
>>> Source)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
>>>         at java.base@16.0.2
>>> /java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
>>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>>   24) Thread[id=366, name=qtp1790869424-366, state=RUNNABLE,
>>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .SelectorImpl.select(SelectorImpl.java:146)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector.select(ManagedSelector.java:190)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>>>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
>>> Source)
>>>         at
>>> app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
>>>         at
>>> app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
>>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>>   25) Thread[id=402, name=CloserThread, state=WAITING,
>>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>>         at java.base@16.0.2/java.lang.Object.wait(Native Method)
>>>         at java.base@16.0.2/java.lang.Object.wait(Object.java:320)
>>>         at
>>> app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2506)
>>>   26) Thread[id=361, name=qtp1790869424-361, state=RUNNABLE,
>>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .SelectorImpl.select(SelectorImpl.java:146)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector.select(ManagedSelector.java:190)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>>>         at app//org.eclipse.jetty.io
>>> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>>>         at
>>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>>>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
>>> Source)
>>>         at
>>> app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
>>>         at
>>> app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
>>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>>   27) Thread[id=393, name=h2sc-30-thread-1, state=RUNNABLE,
>>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>>>         at java.base@16.0.2/sun.nio.ch
>>> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>>>
>>

Re: [JENKINS] Solr-main-Linux (64bit/jdk-16.0.2) - Build # 7849 - Still Unstable!

Posted by Kevin Risden <kr...@apache.org>.
This was caused by the Gradle upgrade. I have a fix locally just running
through tests before pushing.

Kevin Risden


On Sat, Oct 15, 2022 at 8:00 PM Kevin Risden <kr...@apache.org> wrote:

> Most likely one of my commits today caused this. Apparently I didn't run
> the hdfs tests... My guess is commons-text or Gradle. I won't be able to
> look at this until tomorrow at the earliest.
>
> Kevin Risden
>
> On Sat, Oct 15, 2022, 19:55 Policeman Jenkins Server <je...@thetaphi.de>
> wrote:
>
>> Build: https://jenkins.thetaphi.de/job/Solr-main-Linux/7849/
>> Java: 64bit/jdk-16.0.2 -XX:-UseCompressedOops -XX:+UseSerialGC
>>
>> 12 tests failed.
>> FAILED:
>> org.apache.solr.hdfs.cloud.api.collections.HdfsCloudIncrementalBackupTest.classMethod
>>
>> Error Message:
>> java.lang.Exception: Error starting up MiniSolrCloudCluster
>>
>> Stack Trace:
>> java.lang.Exception: Error starting up MiniSolrCloudCluster
>>         at __randomizedtesting.SeedInfo.seed([183F02B0C82BE2D2]:0)
>>         at
>> org.apache.solr.cloud.MiniSolrCloudCluster.checkForExceptions(MiniSolrCloudCluster.java:767)
>>         at
>> org.apache.solr.cloud.MiniSolrCloudCluster.<init>(MiniSolrCloudCluster.java:362)
>>         at
>> org.apache.solr.cloud.MiniSolrCloudCluster$Builder.build(MiniSolrCloudCluster.java:1233)
>>         at
>> org.apache.solr.cloud.MiniSolrCloudCluster$Builder.configure(MiniSolrCloudCluster.java:1205)
>>         at
>> org.apache.solr.hdfs.cloud.api.collections.HdfsCloudIncrementalBackupTest.setupClass(HdfsCloudIncrementalBackupTest.java:129)
>>         at
>> java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native
>> Method)
>>         at
>> java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:78)
>>         at
>> java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>>         at java.base/java.lang.reflect.Method.invoke(Method.java:567)
>>         at
>> com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
>>         at
>> com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:886)
>>         at
>> com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
>>         at
>> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>         at
>> org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
>>         at
>> org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
>>         at
>> com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
>>         at org.junit.rules.RunRules.evaluate(RunRules.java:20)
>>         at
>> org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
>>         at
>> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>         at
>> org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
>>         at
>> com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
>>         at
>> com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
>>         at
>> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>         at
>> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>         at
>> org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
>>         at
>> org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
>>         at
>> org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
>>         at
>> org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
>>         at
>> org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
>>         at org.junit.rules.RunRules.evaluate(RunRules.java:20)
>>         at
>> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>         at
>> com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
>>         at
>> com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
>>         at java.base/java.lang.Thread.run(Thread.java:831)
>>         Suppressed: java.lang.NoClassDefFoundError: Could not initialize
>> class org.apache.hadoop.shaded.com
>> .sun.xml.bind.v2.runtime.reflect.opt.Injector
>>                 at org.apache.hadoop.shaded.com
>> .sun.xml.bind.v2.runtime.reflect.opt.AccessorInjector.prepare(AccessorInjector.java:83)
>>                 at org.apache.hadoop.shaded.com
>> .sun.xml.bind.v2.runtime.reflect.opt.OptimizedAccessorFactory.get(OptimizedAccessorFactory.java:176)
>>                 at org.apache.hadoop.shaded.com
>> .sun.xml.bind.v2.runtime.reflect.Accessor$FieldReflection.optimize(Accessor.java:282)
>>                 at org.apache.hadoop.shaded.com
>> .sun.xml.bind.v2.runtime.property.ArrayProperty.<init>(ArrayProperty.java:69)
>>                 at org.apache.hadoop.shaded.com
>> .sun.xml.bind.v2.runtime.property.ArrayERProperty.<init>(ArrayERProperty.java:88)
>>                 at org.apache.hadoop.shaded.com
>> .sun.xml.bind.v2.runtime.property.ArrayElementProperty.<init>(ArrayElementProperty.java:100)
>>                 at org.apache.hadoop.shaded.com
>> .sun.xml.bind.v2.runtime.property.ArrayElementNodeProperty.<init>(ArrayElementNodeProperty.java:62)
>>                 at
>> java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native
>> Method)
>>                 at
>> java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:78)
>>                 at
>> java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
>>                 at
>> java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499)
>>                 at
>> java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480)
>>                 at org.apache.hadoop.shaded.com
>> .sun.xml.bind.v2.runtime.property.PropertyFactory.create(PropertyFactory.java:128)
>>                 at org.apache.hadoop.shaded.com
>> .sun.xml.bind.v2.runtime.ClassBeanInfoImpl.<init>(ClassBeanInfoImpl.java:183)
>>                 at org.apache.hadoop.shaded.com
>> .sun.xml.bind.v2.runtime.JAXBContextImpl.getOrCreate(JAXBContextImpl.java:532)
>>                 at org.apache.hadoop.shaded.com
>> .sun.xml.bind.v2.runtime.JAXBContextImpl.<init>(JAXBContextImpl.java:347)
>>                 at org.apache.hadoop.shaded.com
>> .sun.xml.bind.v2.runtime.JAXBContextImpl$JAXBContextBuilder.build(JAXBContextImpl.java:1170)
>>                 at org.apache.hadoop.shaded.com
>> .sun.xml.bind.v2.ContextFactory.createContext(ContextFactory.java:145)
>>                 at org.apache.hadoop.shaded.com
>> .sun.xml.bind.v2.ContextFactory.createContext(ContextFactory.java:236)
>>                 at
>> java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native
>> Method)
>>                 at
>> java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:78)
>>                 at
>> java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>>                 at
>> java.base/java.lang.reflect.Method.invoke(Method.java:567)
>>                 at
>> javax.xml.bind.ContextFinder.newInstance(ContextFinder.java:171)
>>                 at
>> javax.xml.bind.ContextFinder.newInstance(ContextFinder.java:129)
>>                 at
>> javax.xml.bind.ContextFinder.find(ContextFinder.java:307)
>>                 at
>> javax.xml.bind.JAXBContext.newInstance(JAXBContext.java:478)
>>                 at
>> javax.xml.bind.JAXBContext.newInstance(JAXBContext.java:435)
>>                 at
>> org.glassfish.jersey.server.wadl.internal.WadlApplicationContextImpl.getJAXBContextFromWadlGenerator(WadlApplicationContextImpl.java:121)
>>                 at
>> org.glassfish.jersey.server.wadl.internal.WadlApplicationContextImpl.isJaxbImplAvailable(WadlApplicationContextImpl.java:270)
>>                 at
>> org.glassfish.jersey.server.wadl.WadlFeature.configure(WadlFeature.java:65)
>>                 at
>> org.glassfish.jersey.model.internal.CommonConfig.configureFeatures(CommonConfig.java:728)
>>                 at
>> org.glassfish.jersey.model.internal.CommonConfig.configureMetaProviders(CommonConfig.java:647)
>>                 at
>> org.glassfish.jersey.server.ResourceConfig.configureMetaProviders(ResourceConfig.java:824)
>>                 at
>> org.glassfish.jersey.server.ApplicationHandler.initialize(ApplicationHandler.java:332)
>>                 at
>> org.glassfish.jersey.server.ApplicationHandler.lambda$initialize$1(ApplicationHandler.java:297)
>>                 at
>> org.glassfish.jersey.internal.Errors.process(Errors.java:292)
>>                 at
>> org.glassfish.jersey.internal.Errors.process(Errors.java:274)
>>                 at
>> org.glassfish.jersey.internal.Errors.processWithException(Errors.java:232)
>>                 at
>> org.glassfish.jersey.server.ApplicationHandler.initialize(ApplicationHandler.java:296)
>>                 at
>> org.glassfish.jersey.server.ApplicationHandler.<init>(ApplicationHandler.java:261)
>>                 at
>> org.glassfish.jersey.server.ApplicationHandler.<init>(ApplicationHandler.java:236)
>>                 at
>> org.apache.solr.core.CoreContainer.load(CoreContainer.java:1086)
>>                 at
>> org.apache.solr.servlet.CoreContainerProvider.createCoreContainer(CoreContainerProvider.java:377)
>>                 at
>> org.apache.solr.servlet.CoreContainerProvider.init(CoreContainerProvider.java:226)
>>                 at
>> org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:402)
>>                 at
>> org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:194)
>>                 at
>> org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:74)
>>                 at
>> org.apache.solr.client.solrj.embedded.JettySolrRunner.retryOnPortBindFailure(JettySolrRunner.java:614)
>>                 at
>> org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:552)
>>                 at
>> org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:523)
>>                 at
>> org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:550)
>>                 at
>> org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:355)
>>                 at
>> java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
>>                 at
>> org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:271)
>>                 at
>> java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
>>                 at
>> java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
>>                 ... 1 more
>>         Suppressed: java.lang.ExceptionInInitializerError
>>                 at org.apache.hadoop.shaded.com
>> .sun.xml.bind.v2.runtime.reflect.opt.AccessorInjector.prepare(AccessorInjector.java:83)
>>                 at org.apache.hadoop.shaded.com
>> .sun.xml.bind.v2.runtime.reflect.opt.OptimizedAccessorFactory.get(OptimizedAccessorFactory.java:176)
>>                 at org.apache.hadoop.shaded.com
>> .sun.xml.bind.v2.runtime.reflect.Accessor$FieldReflection.optimize(Accessor.java:282)
>>                 at org.apache.hadoop.shaded.com
>> .sun.xml.bind.v2.runtime.property.ArrayProperty.<init>(ArrayProperty.java:69)
>>                 at org.apache.hadoop.shaded.com
>> .sun.xml.bind.v2.runtime.property.ArrayERProperty.<init>(ArrayERProperty.java:88)
>>                 at org.apache.hadoop.shaded.com
>> .sun.xml.bind.v2.runtime.property.ArrayElementProperty.<init>(ArrayElementProperty.java:100)
>>                 at org.apache.hadoop.shaded.com
>> .sun.xml.bind.v2.runtime.property.ArrayElementNodeProperty.<init>(ArrayElementNodeProperty.java:62)
>>                 at
>> java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native
>> Method)
>>                 at
>> java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:78)
>>                 at
>> java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
>>                 at
>> java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499)
>>                 at
>> java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480)
>>                 at org.apache.hadoop.shaded.com
>> .sun.xml.bind.v2.runtime.property.PropertyFactory.create(PropertyFactory.java:128)
>>                 at org.apache.hadoop.shaded.com
>> .sun.xml.bind.v2.runtime.ClassBeanInfoImpl.<init>(ClassBeanInfoImpl.java:183)
>>                 at org.apache.hadoop.shaded.com
>> .sun.xml.bind.v2.runtime.JAXBContextImpl.getOrCreate(JAXBContextImpl.java:532)
>>                 at org.apache.hadoop.shaded.com
>> .sun.xml.bind.v2.runtime.JAXBContextImpl.<init>(JAXBContextImpl.java:347)
>>                 at org.apache.hadoop.shaded.com
>> .sun.xml.bind.v2.runtime.JAXBContextImpl$JAXBContextBuilder.build(JAXBContextImpl.java:1170)
>>                 at org.apache.hadoop.shaded.com
>> .sun.xml.bind.v2.ContextFactory.createContext(ContextFactory.java:145)
>>                 at org.apache.hadoop.shaded.com
>> .sun.xml.bind.v2.ContextFactory.createContext(ContextFactory.java:236)
>>                 at
>> java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native
>> Method)
>>                 at
>> java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:78)
>>                 at
>> java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>>                 at
>> java.base/java.lang.reflect.Method.invoke(Method.java:567)
>>                 at
>> javax.xml.bind.ContextFinder.newInstance(ContextFinder.java:171)
>>                 at
>> javax.xml.bind.ContextFinder.newInstance(ContextFinder.java:129)
>>                 at
>> javax.xml.bind.ContextFinder.find(ContextFinder.java:307)
>>                 at
>> javax.xml.bind.JAXBContext.newInstance(JAXBContext.java:478)
>>                 at
>> javax.xml.bind.JAXBContext.newInstance(JAXBContext.java:435)
>>                 at
>> org.glassfish.jersey.server.wadl.internal.WadlApplicationContextImpl.getJAXBContextFromWadlGenerator(WadlApplicationContextImpl.java:121)
>>                 at
>> org.glassfish.jersey.server.wadl.internal.WadlApplicationContextImpl.isJaxbImplAvailable(WadlApplicationContextImpl.java:270)
>>                 at
>> org.glassfish.jersey.server.wadl.WadlFeature.configure(WadlFeature.java:65)
>>                 at
>> org.glassfish.jersey.model.internal.CommonConfig.configureFeatures(CommonConfig.java:728)
>>                 at
>> org.glassfish.jersey.model.internal.CommonConfig.configureMetaProviders(CommonConfig.java:647)
>>                 at
>> org.glassfish.jersey.server.ResourceConfig.configureMetaProviders(ResourceConfig.java:824)
>>                 at
>> org.glassfish.jersey.server.ApplicationHandler.initialize(ApplicationHandler.java:332)
>>                 at
>> org.glassfish.jersey.server.ApplicationHandler.lambda$initialize$1(ApplicationHandler.java:297)
>>                 at
>> org.glassfish.jersey.internal.Errors.process(Errors.java:292)
>>                 at
>> org.glassfish.jersey.internal.Errors.process(Errors.java:274)
>>                 at
>> org.glassfish.jersey.internal.Errors.processWithException(Errors.java:232)
>>                 at
>> org.glassfish.jersey.server.ApplicationHandler.initialize(ApplicationHandler.java:296)
>>                 at
>> org.glassfish.jersey.server.ApplicationHandler.<init>(ApplicationHandler.java:261)
>>                 at
>> org.glassfish.jersey.server.ApplicationHandler.<init>(ApplicationHandler.java:236)
>>                 at
>> org.apache.solr.core.CoreContainer.load(CoreContainer.java:1086)
>>                 at
>> org.apache.solr.servlet.CoreContainerProvider.createCoreContainer(CoreContainerProvider.java:377)
>>                 at
>> org.apache.solr.servlet.CoreContainerProvider.init(CoreContainerProvider.java:226)
>>                 at
>> org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:402)
>>                 at
>> org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:194)
>>                 at
>> org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:74)
>>                 at
>> org.apache.solr.client.solrj.embedded.JettySolrRunner.retryOnPortBindFailure(JettySolrRunner.java:614)
>>                 at
>> org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:552)
>>                 at
>> org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:523)
>>                 at
>> org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:550)
>>                 at
>> org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:355)
>>                 at
>> java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
>>                 at
>> org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:271)
>>                 at
>> java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
>>                 at
>> java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
>>                 ... 1 more
>>         Caused by: java.lang.reflect.InaccessibleObjectException: Unable
>> to make protected final java.lang.Class
>> java.lang.ClassLoader.defineClass(java.lang.String,byte[],int,int) throws
>> java.lang.ClassFormatError accessible: module java.base does not "opens
>> java.lang" to unnamed module @77f1baf5
>>                 at
>> java.base/java.lang.reflect.AccessibleObject.checkCanSetAccessible(AccessibleObject.java:357)
>>                 at
>> java.base/java.lang.reflect.AccessibleObject.checkCanSetAccessible(AccessibleObject.java:297)
>>                 at
>> java.base/java.lang.reflect.Method.checkCanSetAccessible(Method.java:199)
>>                 at
>> java.base/java.lang.reflect.Method.setAccessible(Method.java:193)
>>                 at org.apache.hadoop.shaded.com
>> .sun.xml.bind.v2.runtime.reflect.opt.Injector$1.run(Injector.java:177)
>>                 at org.apache.hadoop.shaded.com
>> .sun.xml.bind.v2.runtime.reflect.opt.Injector$1.run(Injector.java:174)
>>                 at
>> java.base/java.security.AccessController.doPrivileged(AccessController.java:312)
>>                 at org.apache.hadoop.shaded.com
>> .sun.xml.bind.v2.runtime.reflect.opt.Injector.<clinit>(Injector.java:172)
>>                 ... 58 more
>>
>>
>> FAILED:
>> org.apache.solr.hdfs.cloud.api.collections.HdfsCloudIncrementalBackupTest.classMethod
>>
>> Error Message:
>> com.carrotsearch.randomizedtesting.ThreadLeakError: 46 threads leaked
>> from SUITE scope at
>> org.apache.solr.hdfs.cloud.api.collections.HdfsCloudIncrementalBackupTest:
>>    1) Thread[id=401, name=CloserThread, state=WAITING,
>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>         at java.base@16.0.2/java.lang.Object.wait(Native Method)
>>         at java.base@16.0.2/java.lang.Object.wait(Object.java:320)
>>         at
>> app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2506)
>>    2) Thread[id=372, name=Session-HouseKeeper-74f5e0-1,
>> state=TIMED_WAITING, group=TGRP-HdfsCloudIncrementalBackupTest]
>>         at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
>>         at java.base@16.0.2
>> /java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
>>         at java.base@16.0.2
>> /java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
>>         at java.base@16.0.2
>> /java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
>>         at java.base@16.0.2
>> /java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
>>         at java.base@16.0.2
>> /java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
>>         at java.base@16.0.2
>> /java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
>>         at java.base@16.0.2
>> /java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>    3) Thread[id=400,
>> name=OverseerCollectionConfigSetProcessor-72262794004201478-127.0.0.1:46753_solr-n_0000000000,
>> state=WAITING, group=Overseer collection creation process.]
>>         at java.base@16.0.2/java.lang.Object.wait(Native Method)
>>         at java.base@16.0.2/java.lang.Object.wait(Object.java:320)
>>         at
>> app//org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1590)
>>         at
>> app//org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1562)
>>         at
>> app//org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1970)
>>         at
>> app//org.apache.solr.common.cloud.SolrZkClient.lambda$getData$6(SolrZkClient.java:432)
>>         at
>> app//org.apache.solr.common.cloud.SolrZkClient$$Lambda$382/0x00000008013d7858.execute(Unknown
>> Source)
>>         at
>> app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:72)
>>         at
>> app//org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:432)
>>         at
>> app//org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:486)
>>         at
>> app//org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:232)
>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>    4) Thread[id=367, name=qtp1790869424-367, state=RUNNABLE,
>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>>         at java.base@16.0.2/sun.nio.ch
>> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>>         at java.base@16.0.2/sun.nio.ch
>> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>>         at java.base@16.0.2/sun.nio.ch
>> .SelectorImpl.select(SelectorImpl.java:146)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector.select(ManagedSelector.java:190)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
>> Source)
>>         at
>> app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
>>         at
>> app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>    5) Thread[id=360, name=qtp577348919-360, state=RUNNABLE,
>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>>         at java.base@16.0.2/sun.nio.ch
>> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>>         at java.base@16.0.2/sun.nio.ch
>> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>>         at java.base@16.0.2/sun.nio.ch
>> .SelectorImpl.select(SelectorImpl.java:146)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector.select(ManagedSelector.java:190)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
>> Source)
>>         at
>> app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
>>         at
>> app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>    6) Thread[id=404, name=coreContainerWorkExecutor-18-thread-1,
>> state=TIMED_WAITING, group=TGRP-HdfsCloudIncrementalBackupTest]
>>         at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
>>         at java.base@16.0.2
>> /java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
>>         at java.base@16.0.2
>> /java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
>>         at java.base@16.0.2
>> /java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
>>         at java.base@16.0.2
>> /java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
>>         at java.base@16.0.2
>> /java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
>>         at java.base@16.0.2
>> /java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
>>         at java.base@16.0.2
>> /java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>    7) Thread[id=383, name=httpShardExecutor-21-thread-1, state=RUNNABLE,
>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>>         at java.base@16.0.2/sun.nio.ch
>> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>>         at java.base@16.0.2/sun.nio.ch
>> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>>         at java.base@16.0.2/sun.nio.ch
>> .SelectorImpl.select(SelectorImpl.java:146)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector.select(ManagedSelector.java:190)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
>> Source)
>>         at
>> app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:271)
>>         at
>> app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$349/0x0000000801384860.run(Unknown
>> Source)
>>         at java.base@16.0.2
>> /java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
>>         at java.base@16.0.2
>> /java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>    8) Thread[id=364, name=qtp1790869424-364, state=RUNNABLE,
>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>>         at java.base@16.0.2/sun.nio.ch
>> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>>         at java.base@16.0.2/sun.nio.ch
>> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>>         at java.base@16.0.2/sun.nio.ch
>> .SelectorImpl.select(SelectorImpl.java:146)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector.select(ManagedSelector.java:190)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
>> Source)
>>         at
>> app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
>>         at
>> app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>    9) Thread[id=428, name=zkCallback-32-thread-4, state=TIMED_WAITING,
>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>         at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
>>         at java.base@16.0.2
>> /java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
>>         at java.base@16.0.2
>> /java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
>>         at java.base@16.0.2
>> /java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
>>         at java.base@16.0.2
>> /java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
>>         at java.base@16.0.2
>> /java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
>>         at java.base@16.0.2
>> /java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
>>         at java.base@16.0.2
>> /java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>   10) Thread[id=418, name=zkCallback-32-thread-2, state=TIMED_WAITING,
>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>         at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
>>         at java.base@16.0.2
>> /java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
>>         at java.base@16.0.2
>> /java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
>>         at java.base@16.0.2
>> /java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
>>         at java.base@16.0.2
>> /java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
>>         at java.base@16.0.2
>> /java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
>>         at java.base@16.0.2
>> /java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
>>         at java.base@16.0.2
>> /java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>   11) Thread[id=365, name=qtp577348919-365, state=RUNNABLE,
>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>>         at java.base@16.0.2/sun.nio.ch
>> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>>         at java.base@16.0.2/sun.nio.ch
>> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>>         at java.base@16.0.2/sun.nio.ch
>> .SelectorImpl.select(SelectorImpl.java:146)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector.select(ManagedSelector.java:190)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
>> Source)
>>         at
>> app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
>>         at
>> app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>   12) Thread[id=369, name=qtp1790869424-369, state=RUNNABLE,
>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>>         at java.base@16.0.2/sun.nio.ch
>> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>>         at java.base@16.0.2/sun.nio.ch
>> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>>         at java.base@16.0.2/sun.nio.ch
>> .SelectorImpl.select(SelectorImpl.java:146)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector.select(ManagedSelector.java:190)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
>> Source)
>>         at
>> app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
>>         at
>> app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>   13) Thread[id=387, name=httpShardExecutor-26-thread-1, state=RUNNABLE,
>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>>         at java.base@16.0.2/sun.nio.ch
>> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>>         at java.base@16.0.2/sun.nio.ch
>> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>>         at java.base@16.0.2/sun.nio.ch
>> .SelectorImpl.select(SelectorImpl.java:146)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector.select(ManagedSelector.java:190)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
>> Source)
>>         at
>> app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:271)
>>         at
>> app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$349/0x0000000801384860.run(Unknown
>> Source)
>>         at java.base@16.0.2
>> /java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
>>         at java.base@16.0.2
>> /java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>   14) Thread[id=355, name=qtp577348919-355, state=RUNNABLE,
>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>>         at java.base@16.0.2/sun.nio.ch
>> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>>         at java.base@16.0.2/sun.nio.ch
>> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>>         at java.base@16.0.2/sun.nio.ch
>> .SelectorImpl.select(SelectorImpl.java:146)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector.select(ManagedSelector.java:190)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
>> Source)
>>         at
>> app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
>>         at
>> app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>   15) Thread[id=396, name=zkConnectionManagerCallback-33-thread-1,
>> state=WAITING, group=TGRP-HdfsCloudIncrementalBackupTest]
>>         at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
>>         at java.base@16.0.2
>> /java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
>>         at java.base@16.0.2
>> /java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
>>         at java.base@16.0.2
>> /java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
>>         at java.base@16.0.2
>> /java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
>>         at java.base@16.0.2
>> /java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
>>         at java.base@16.0.2
>> /java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
>>         at java.base@16.0.2
>> /java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
>>         at java.base@16.0.2
>> /java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>   16) Thread[id=362, name=qtp577348919-362, state=RUNNABLE,
>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>>         at java.base@16.0.2/sun.nio.ch
>> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>>         at java.base@16.0.2/sun.nio.ch
>> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>>         at java.base@16.0.2/sun.nio.ch
>> .SelectorImpl.select(SelectorImpl.java:146)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector.select(ManagedSelector.java:190)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
>> Source)
>>         at
>> app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
>>         at
>> app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>   17) Thread[id=363, name=qtp577348919-363, state=RUNNABLE,
>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>>         at java.base@16.0.2/sun.nio.ch
>> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>>         at java.base@16.0.2/sun.nio.ch
>> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>>         at java.base@16.0.2/sun.nio.ch
>> .SelectorImpl.select(SelectorImpl.java:146)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector.select(ManagedSelector.java:190)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
>> Source)
>>         at
>> app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
>>         at
>> app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>   18) Thread[id=389, name=jetty-launcher-9-thread-2-EventThread,
>> state=WAITING, group=TGRP-HdfsCloudIncrementalBackupTest]
>>         at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
>>         at java.base@16.0.2
>> /java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
>>         at java.base@16.0.2
>> /java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
>>         at java.base@16.0.2
>> /java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
>>         at java.base@16.0.2
>> /java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
>>         at java.base@16.0.2
>> /java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
>>         at
>> app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:549)
>>   19) Thread[id=374,
>> name=qtp1790869424-374-acceptor-0@31e11e51-ServerConnector@442851d5{ssl,
>> (ssl, alpn, http/1.1, h2)}{127.0.0.1:44953}, state=RUNNABLE,
>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>         at java.base@16.0.2/sun.nio.ch.Net.accept(Native Method)
>>         at java.base@16.0.2/sun.nio.ch
>> .ServerSocketChannelImpl.implAccept(ServerSocketChannelImpl.java:424)
>>         at java.base@16.0.2/sun.nio.ch
>> .ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:390)
>>         at
>> app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388)
>>         at
>> app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:704)
>>         at
>> app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
>>         at
>> app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>   20) Thread[id=356, name=qtp1790869424-356, state=RUNNABLE,
>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>>         at java.base@16.0.2/sun.nio.ch
>> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>>         at java.base@16.0.2/sun.nio.ch
>> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>>         at java.base@16.0.2/sun.nio.ch
>> .SelectorImpl.select(SelectorImpl.java:146)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector.select(ManagedSelector.java:190)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
>> Source)
>>         at
>> app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
>>         at
>> app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>   21) Thread[id=417, name=zkCallback-27-thread-2, state=TIMED_WAITING,
>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>         at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
>>         at java.base@16.0.2
>> /java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
>>         at java.base@16.0.2
>> /java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
>>         at java.base@16.0.2
>> /java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
>>         at java.base@16.0.2
>> /java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
>>         at java.base@16.0.2
>> /java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
>>         at java.base@16.0.2
>> /java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
>>         at java.base@16.0.2
>> /java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>   22) Thread[id=359, name=qtp577348919-359, state=RUNNABLE,
>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>>         at java.base@16.0.2/sun.nio.ch
>> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>>         at java.base@16.0.2/sun.nio.ch
>> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>>         at java.base@16.0.2/sun.nio.ch
>> .SelectorImpl.select(SelectorImpl.java:146)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector.select(ManagedSelector.java:190)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
>> Source)
>>         at
>> app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
>>         at
>> app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>   23) Thread[id=386, name=h2sc-23-thread-1, state=RUNNABLE,
>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>>         at java.base@16.0.2/sun.nio.ch
>> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>>         at java.base@16.0.2/sun.nio.ch
>> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>>         at java.base@16.0.2/sun.nio.ch
>> .SelectorImpl.select(SelectorImpl.java:146)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector.select(ManagedSelector.java:190)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
>> Source)
>>         at
>> app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:271)
>>         at
>> app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$349/0x0000000801384860.run(Unknown
>> Source)
>>         at java.base@16.0.2
>> /java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
>>         at java.base@16.0.2
>> /java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>   24) Thread[id=366, name=qtp1790869424-366, state=RUNNABLE,
>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>>         at java.base@16.0.2/sun.nio.ch
>> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>>         at java.base@16.0.2/sun.nio.ch
>> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>>         at java.base@16.0.2/sun.nio.ch
>> .SelectorImpl.select(SelectorImpl.java:146)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector.select(ManagedSelector.java:190)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
>> Source)
>>         at
>> app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
>>         at
>> app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>   25) Thread[id=402, name=CloserThread, state=WAITING,
>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>         at java.base@16.0.2/java.lang.Object.wait(Native Method)
>>         at java.base@16.0.2/java.lang.Object.wait(Object.java:320)
>>         at
>> app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2506)
>>   26) Thread[id=361, name=qtp1790869424-361, state=RUNNABLE,
>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>>         at java.base@16.0.2/sun.nio.ch
>> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>>         at java.base@16.0.2/sun.nio.ch
>> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>>         at java.base@16.0.2/sun.nio.ch
>> .SelectorImpl.select(SelectorImpl.java:146)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector.select(ManagedSelector.java:190)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
>> Source)
>>         at
>> app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
>>         at
>> app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>   27) Thread[id=393, name=h2sc-30-thread-1, state=RUNNABLE,
>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>>         at java.base@16.0.2/sun.nio.ch
>> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>>         at java.base@16.0.2/sun.nio.ch
>> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>>
>

Re: [JENKINS] Solr-main-Linux (64bit/jdk-16.0.2) - Build # 7849 - Still Unstable!

Posted by Kevin Risden <kr...@apache.org>.
This was caused by the Gradle upgrade. I have a fix locally just running
through tests before pushing.

Kevin Risden


On Sat, Oct 15, 2022 at 8:00 PM Kevin Risden <kr...@apache.org> wrote:

> Most likely one of my commits today caused this. Apparently I didn't run
> the hdfs tests... My guess is commons-text or Gradle. I won't be able to
> look at this until tomorrow at the earliest.
>
> Kevin Risden
>
> On Sat, Oct 15, 2022, 19:55 Policeman Jenkins Server <je...@thetaphi.de>
> wrote:
>
>> Build: https://jenkins.thetaphi.de/job/Solr-main-Linux/7849/
>> Java: 64bit/jdk-16.0.2 -XX:-UseCompressedOops -XX:+UseSerialGC
>>
>> 12 tests failed.
>> FAILED:
>> org.apache.solr.hdfs.cloud.api.collections.HdfsCloudIncrementalBackupTest.classMethod
>>
>> Error Message:
>> java.lang.Exception: Error starting up MiniSolrCloudCluster
>>
>> Stack Trace:
>> java.lang.Exception: Error starting up MiniSolrCloudCluster
>>         at __randomizedtesting.SeedInfo.seed([183F02B0C82BE2D2]:0)
>>         at
>> org.apache.solr.cloud.MiniSolrCloudCluster.checkForExceptions(MiniSolrCloudCluster.java:767)
>>         at
>> org.apache.solr.cloud.MiniSolrCloudCluster.<init>(MiniSolrCloudCluster.java:362)
>>         at
>> org.apache.solr.cloud.MiniSolrCloudCluster$Builder.build(MiniSolrCloudCluster.java:1233)
>>         at
>> org.apache.solr.cloud.MiniSolrCloudCluster$Builder.configure(MiniSolrCloudCluster.java:1205)
>>         at
>> org.apache.solr.hdfs.cloud.api.collections.HdfsCloudIncrementalBackupTest.setupClass(HdfsCloudIncrementalBackupTest.java:129)
>>         at
>> java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native
>> Method)
>>         at
>> java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:78)
>>         at
>> java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>>         at java.base/java.lang.reflect.Method.invoke(Method.java:567)
>>         at
>> com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1758)
>>         at
>> com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:886)
>>         at
>> com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:902)
>>         at
>> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>         at
>> org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
>>         at
>> org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
>>         at
>> com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:80)
>>         at org.junit.rules.RunRules.evaluate(RunRules.java:20)
>>         at
>> org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
>>         at
>> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>         at
>> org.apache.lucene.tests.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:38)
>>         at
>> com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
>>         at
>> com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
>>         at
>> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>         at
>> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>         at
>> org.apache.lucene.tests.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
>>         at
>> org.apache.lucene.tests.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:43)
>>         at
>> org.apache.lucene.tests.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:44)
>>         at
>> org.apache.lucene.tests.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:60)
>>         at
>> org.apache.lucene.tests.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:47)
>>         at org.junit.rules.RunRules.evaluate(RunRules.java:20)
>>         at
>> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>         at
>> com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:390)
>>         at
>> com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:850)
>>         at java.base/java.lang.Thread.run(Thread.java:831)
>>         Suppressed: java.lang.NoClassDefFoundError: Could not initialize
>> class org.apache.hadoop.shaded.com
>> .sun.xml.bind.v2.runtime.reflect.opt.Injector
>>                 at org.apache.hadoop.shaded.com
>> .sun.xml.bind.v2.runtime.reflect.opt.AccessorInjector.prepare(AccessorInjector.java:83)
>>                 at org.apache.hadoop.shaded.com
>> .sun.xml.bind.v2.runtime.reflect.opt.OptimizedAccessorFactory.get(OptimizedAccessorFactory.java:176)
>>                 at org.apache.hadoop.shaded.com
>> .sun.xml.bind.v2.runtime.reflect.Accessor$FieldReflection.optimize(Accessor.java:282)
>>                 at org.apache.hadoop.shaded.com
>> .sun.xml.bind.v2.runtime.property.ArrayProperty.<init>(ArrayProperty.java:69)
>>                 at org.apache.hadoop.shaded.com
>> .sun.xml.bind.v2.runtime.property.ArrayERProperty.<init>(ArrayERProperty.java:88)
>>                 at org.apache.hadoop.shaded.com
>> .sun.xml.bind.v2.runtime.property.ArrayElementProperty.<init>(ArrayElementProperty.java:100)
>>                 at org.apache.hadoop.shaded.com
>> .sun.xml.bind.v2.runtime.property.ArrayElementNodeProperty.<init>(ArrayElementNodeProperty.java:62)
>>                 at
>> java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native
>> Method)
>>                 at
>> java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:78)
>>                 at
>> java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
>>                 at
>> java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499)
>>                 at
>> java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480)
>>                 at org.apache.hadoop.shaded.com
>> .sun.xml.bind.v2.runtime.property.PropertyFactory.create(PropertyFactory.java:128)
>>                 at org.apache.hadoop.shaded.com
>> .sun.xml.bind.v2.runtime.ClassBeanInfoImpl.<init>(ClassBeanInfoImpl.java:183)
>>                 at org.apache.hadoop.shaded.com
>> .sun.xml.bind.v2.runtime.JAXBContextImpl.getOrCreate(JAXBContextImpl.java:532)
>>                 at org.apache.hadoop.shaded.com
>> .sun.xml.bind.v2.runtime.JAXBContextImpl.<init>(JAXBContextImpl.java:347)
>>                 at org.apache.hadoop.shaded.com
>> .sun.xml.bind.v2.runtime.JAXBContextImpl$JAXBContextBuilder.build(JAXBContextImpl.java:1170)
>>                 at org.apache.hadoop.shaded.com
>> .sun.xml.bind.v2.ContextFactory.createContext(ContextFactory.java:145)
>>                 at org.apache.hadoop.shaded.com
>> .sun.xml.bind.v2.ContextFactory.createContext(ContextFactory.java:236)
>>                 at
>> java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native
>> Method)
>>                 at
>> java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:78)
>>                 at
>> java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>>                 at
>> java.base/java.lang.reflect.Method.invoke(Method.java:567)
>>                 at
>> javax.xml.bind.ContextFinder.newInstance(ContextFinder.java:171)
>>                 at
>> javax.xml.bind.ContextFinder.newInstance(ContextFinder.java:129)
>>                 at
>> javax.xml.bind.ContextFinder.find(ContextFinder.java:307)
>>                 at
>> javax.xml.bind.JAXBContext.newInstance(JAXBContext.java:478)
>>                 at
>> javax.xml.bind.JAXBContext.newInstance(JAXBContext.java:435)
>>                 at
>> org.glassfish.jersey.server.wadl.internal.WadlApplicationContextImpl.getJAXBContextFromWadlGenerator(WadlApplicationContextImpl.java:121)
>>                 at
>> org.glassfish.jersey.server.wadl.internal.WadlApplicationContextImpl.isJaxbImplAvailable(WadlApplicationContextImpl.java:270)
>>                 at
>> org.glassfish.jersey.server.wadl.WadlFeature.configure(WadlFeature.java:65)
>>                 at
>> org.glassfish.jersey.model.internal.CommonConfig.configureFeatures(CommonConfig.java:728)
>>                 at
>> org.glassfish.jersey.model.internal.CommonConfig.configureMetaProviders(CommonConfig.java:647)
>>                 at
>> org.glassfish.jersey.server.ResourceConfig.configureMetaProviders(ResourceConfig.java:824)
>>                 at
>> org.glassfish.jersey.server.ApplicationHandler.initialize(ApplicationHandler.java:332)
>>                 at
>> org.glassfish.jersey.server.ApplicationHandler.lambda$initialize$1(ApplicationHandler.java:297)
>>                 at
>> org.glassfish.jersey.internal.Errors.process(Errors.java:292)
>>                 at
>> org.glassfish.jersey.internal.Errors.process(Errors.java:274)
>>                 at
>> org.glassfish.jersey.internal.Errors.processWithException(Errors.java:232)
>>                 at
>> org.glassfish.jersey.server.ApplicationHandler.initialize(ApplicationHandler.java:296)
>>                 at
>> org.glassfish.jersey.server.ApplicationHandler.<init>(ApplicationHandler.java:261)
>>                 at
>> org.glassfish.jersey.server.ApplicationHandler.<init>(ApplicationHandler.java:236)
>>                 at
>> org.apache.solr.core.CoreContainer.load(CoreContainer.java:1086)
>>                 at
>> org.apache.solr.servlet.CoreContainerProvider.createCoreContainer(CoreContainerProvider.java:377)
>>                 at
>> org.apache.solr.servlet.CoreContainerProvider.init(CoreContainerProvider.java:226)
>>                 at
>> org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:402)
>>                 at
>> org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:194)
>>                 at
>> org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:74)
>>                 at
>> org.apache.solr.client.solrj.embedded.JettySolrRunner.retryOnPortBindFailure(JettySolrRunner.java:614)
>>                 at
>> org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:552)
>>                 at
>> org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:523)
>>                 at
>> org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:550)
>>                 at
>> org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:355)
>>                 at
>> java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
>>                 at
>> org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:271)
>>                 at
>> java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
>>                 at
>> java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
>>                 ... 1 more
>>         Suppressed: java.lang.ExceptionInInitializerError
>>                 at org.apache.hadoop.shaded.com
>> .sun.xml.bind.v2.runtime.reflect.opt.AccessorInjector.prepare(AccessorInjector.java:83)
>>                 at org.apache.hadoop.shaded.com
>> .sun.xml.bind.v2.runtime.reflect.opt.OptimizedAccessorFactory.get(OptimizedAccessorFactory.java:176)
>>                 at org.apache.hadoop.shaded.com
>> .sun.xml.bind.v2.runtime.reflect.Accessor$FieldReflection.optimize(Accessor.java:282)
>>                 at org.apache.hadoop.shaded.com
>> .sun.xml.bind.v2.runtime.property.ArrayProperty.<init>(ArrayProperty.java:69)
>>                 at org.apache.hadoop.shaded.com
>> .sun.xml.bind.v2.runtime.property.ArrayERProperty.<init>(ArrayERProperty.java:88)
>>                 at org.apache.hadoop.shaded.com
>> .sun.xml.bind.v2.runtime.property.ArrayElementProperty.<init>(ArrayElementProperty.java:100)
>>                 at org.apache.hadoop.shaded.com
>> .sun.xml.bind.v2.runtime.property.ArrayElementNodeProperty.<init>(ArrayElementNodeProperty.java:62)
>>                 at
>> java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native
>> Method)
>>                 at
>> java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:78)
>>                 at
>> java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
>>                 at
>> java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499)
>>                 at
>> java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480)
>>                 at org.apache.hadoop.shaded.com
>> .sun.xml.bind.v2.runtime.property.PropertyFactory.create(PropertyFactory.java:128)
>>                 at org.apache.hadoop.shaded.com
>> .sun.xml.bind.v2.runtime.ClassBeanInfoImpl.<init>(ClassBeanInfoImpl.java:183)
>>                 at org.apache.hadoop.shaded.com
>> .sun.xml.bind.v2.runtime.JAXBContextImpl.getOrCreate(JAXBContextImpl.java:532)
>>                 at org.apache.hadoop.shaded.com
>> .sun.xml.bind.v2.runtime.JAXBContextImpl.<init>(JAXBContextImpl.java:347)
>>                 at org.apache.hadoop.shaded.com
>> .sun.xml.bind.v2.runtime.JAXBContextImpl$JAXBContextBuilder.build(JAXBContextImpl.java:1170)
>>                 at org.apache.hadoop.shaded.com
>> .sun.xml.bind.v2.ContextFactory.createContext(ContextFactory.java:145)
>>                 at org.apache.hadoop.shaded.com
>> .sun.xml.bind.v2.ContextFactory.createContext(ContextFactory.java:236)
>>                 at
>> java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native
>> Method)
>>                 at
>> java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:78)
>>                 at
>> java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>>                 at
>> java.base/java.lang.reflect.Method.invoke(Method.java:567)
>>                 at
>> javax.xml.bind.ContextFinder.newInstance(ContextFinder.java:171)
>>                 at
>> javax.xml.bind.ContextFinder.newInstance(ContextFinder.java:129)
>>                 at
>> javax.xml.bind.ContextFinder.find(ContextFinder.java:307)
>>                 at
>> javax.xml.bind.JAXBContext.newInstance(JAXBContext.java:478)
>>                 at
>> javax.xml.bind.JAXBContext.newInstance(JAXBContext.java:435)
>>                 at
>> org.glassfish.jersey.server.wadl.internal.WadlApplicationContextImpl.getJAXBContextFromWadlGenerator(WadlApplicationContextImpl.java:121)
>>                 at
>> org.glassfish.jersey.server.wadl.internal.WadlApplicationContextImpl.isJaxbImplAvailable(WadlApplicationContextImpl.java:270)
>>                 at
>> org.glassfish.jersey.server.wadl.WadlFeature.configure(WadlFeature.java:65)
>>                 at
>> org.glassfish.jersey.model.internal.CommonConfig.configureFeatures(CommonConfig.java:728)
>>                 at
>> org.glassfish.jersey.model.internal.CommonConfig.configureMetaProviders(CommonConfig.java:647)
>>                 at
>> org.glassfish.jersey.server.ResourceConfig.configureMetaProviders(ResourceConfig.java:824)
>>                 at
>> org.glassfish.jersey.server.ApplicationHandler.initialize(ApplicationHandler.java:332)
>>                 at
>> org.glassfish.jersey.server.ApplicationHandler.lambda$initialize$1(ApplicationHandler.java:297)
>>                 at
>> org.glassfish.jersey.internal.Errors.process(Errors.java:292)
>>                 at
>> org.glassfish.jersey.internal.Errors.process(Errors.java:274)
>>                 at
>> org.glassfish.jersey.internal.Errors.processWithException(Errors.java:232)
>>                 at
>> org.glassfish.jersey.server.ApplicationHandler.initialize(ApplicationHandler.java:296)
>>                 at
>> org.glassfish.jersey.server.ApplicationHandler.<init>(ApplicationHandler.java:261)
>>                 at
>> org.glassfish.jersey.server.ApplicationHandler.<init>(ApplicationHandler.java:236)
>>                 at
>> org.apache.solr.core.CoreContainer.load(CoreContainer.java:1086)
>>                 at
>> org.apache.solr.servlet.CoreContainerProvider.createCoreContainer(CoreContainerProvider.java:377)
>>                 at
>> org.apache.solr.servlet.CoreContainerProvider.init(CoreContainerProvider.java:226)
>>                 at
>> org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:402)
>>                 at
>> org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:194)
>>                 at
>> org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:74)
>>                 at
>> org.apache.solr.client.solrj.embedded.JettySolrRunner.retryOnPortBindFailure(JettySolrRunner.java:614)
>>                 at
>> org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:552)
>>                 at
>> org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:523)
>>                 at
>> org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:550)
>>                 at
>> org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:355)
>>                 at
>> java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
>>                 at
>> org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:271)
>>                 at
>> java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
>>                 at
>> java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
>>                 ... 1 more
>>         Caused by: java.lang.reflect.InaccessibleObjectException: Unable
>> to make protected final java.lang.Class
>> java.lang.ClassLoader.defineClass(java.lang.String,byte[],int,int) throws
>> java.lang.ClassFormatError accessible: module java.base does not "opens
>> java.lang" to unnamed module @77f1baf5
>>                 at
>> java.base/java.lang.reflect.AccessibleObject.checkCanSetAccessible(AccessibleObject.java:357)
>>                 at
>> java.base/java.lang.reflect.AccessibleObject.checkCanSetAccessible(AccessibleObject.java:297)
>>                 at
>> java.base/java.lang.reflect.Method.checkCanSetAccessible(Method.java:199)
>>                 at
>> java.base/java.lang.reflect.Method.setAccessible(Method.java:193)
>>                 at org.apache.hadoop.shaded.com
>> .sun.xml.bind.v2.runtime.reflect.opt.Injector$1.run(Injector.java:177)
>>                 at org.apache.hadoop.shaded.com
>> .sun.xml.bind.v2.runtime.reflect.opt.Injector$1.run(Injector.java:174)
>>                 at
>> java.base/java.security.AccessController.doPrivileged(AccessController.java:312)
>>                 at org.apache.hadoop.shaded.com
>> .sun.xml.bind.v2.runtime.reflect.opt.Injector.<clinit>(Injector.java:172)
>>                 ... 58 more
>>
>>
>> FAILED:
>> org.apache.solr.hdfs.cloud.api.collections.HdfsCloudIncrementalBackupTest.classMethod
>>
>> Error Message:
>> com.carrotsearch.randomizedtesting.ThreadLeakError: 46 threads leaked
>> from SUITE scope at
>> org.apache.solr.hdfs.cloud.api.collections.HdfsCloudIncrementalBackupTest:
>>    1) Thread[id=401, name=CloserThread, state=WAITING,
>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>         at java.base@16.0.2/java.lang.Object.wait(Native Method)
>>         at java.base@16.0.2/java.lang.Object.wait(Object.java:320)
>>         at
>> app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2506)
>>    2) Thread[id=372, name=Session-HouseKeeper-74f5e0-1,
>> state=TIMED_WAITING, group=TGRP-HdfsCloudIncrementalBackupTest]
>>         at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
>>         at java.base@16.0.2
>> /java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
>>         at java.base@16.0.2
>> /java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1661)
>>         at java.base@16.0.2
>> /java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
>>         at java.base@16.0.2
>> /java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
>>         at java.base@16.0.2
>> /java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
>>         at java.base@16.0.2
>> /java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
>>         at java.base@16.0.2
>> /java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>    3) Thread[id=400,
>> name=OverseerCollectionConfigSetProcessor-72262794004201478-127.0.0.1:46753_solr-n_0000000000,
>> state=WAITING, group=Overseer collection creation process.]
>>         at java.base@16.0.2/java.lang.Object.wait(Native Method)
>>         at java.base@16.0.2/java.lang.Object.wait(Object.java:320)
>>         at
>> app//org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1590)
>>         at
>> app//org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1562)
>>         at
>> app//org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1970)
>>         at
>> app//org.apache.solr.common.cloud.SolrZkClient.lambda$getData$6(SolrZkClient.java:432)
>>         at
>> app//org.apache.solr.common.cloud.SolrZkClient$$Lambda$382/0x00000008013d7858.execute(Unknown
>> Source)
>>         at
>> app//org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:72)
>>         at
>> app//org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:432)
>>         at
>> app//org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:486)
>>         at
>> app//org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:232)
>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>    4) Thread[id=367, name=qtp1790869424-367, state=RUNNABLE,
>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>>         at java.base@16.0.2/sun.nio.ch
>> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>>         at java.base@16.0.2/sun.nio.ch
>> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>>         at java.base@16.0.2/sun.nio.ch
>> .SelectorImpl.select(SelectorImpl.java:146)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector.select(ManagedSelector.java:190)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
>> Source)
>>         at
>> app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
>>         at
>> app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>    5) Thread[id=360, name=qtp577348919-360, state=RUNNABLE,
>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>>         at java.base@16.0.2/sun.nio.ch
>> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>>         at java.base@16.0.2/sun.nio.ch
>> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>>         at java.base@16.0.2/sun.nio.ch
>> .SelectorImpl.select(SelectorImpl.java:146)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector.select(ManagedSelector.java:190)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
>> Source)
>>         at
>> app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
>>         at
>> app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>    6) Thread[id=404, name=coreContainerWorkExecutor-18-thread-1,
>> state=TIMED_WAITING, group=TGRP-HdfsCloudIncrementalBackupTest]
>>         at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
>>         at java.base@16.0.2
>> /java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
>>         at java.base@16.0.2
>> /java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
>>         at java.base@16.0.2
>> /java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
>>         at java.base@16.0.2
>> /java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
>>         at java.base@16.0.2
>> /java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
>>         at java.base@16.0.2
>> /java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
>>         at java.base@16.0.2
>> /java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>    7) Thread[id=383, name=httpShardExecutor-21-thread-1, state=RUNNABLE,
>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>>         at java.base@16.0.2/sun.nio.ch
>> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>>         at java.base@16.0.2/sun.nio.ch
>> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>>         at java.base@16.0.2/sun.nio.ch
>> .SelectorImpl.select(SelectorImpl.java:146)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector.select(ManagedSelector.java:190)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
>> Source)
>>         at
>> app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:271)
>>         at
>> app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$349/0x0000000801384860.run(Unknown
>> Source)
>>         at java.base@16.0.2
>> /java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
>>         at java.base@16.0.2
>> /java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>    8) Thread[id=364, name=qtp1790869424-364, state=RUNNABLE,
>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>>         at java.base@16.0.2/sun.nio.ch
>> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>>         at java.base@16.0.2/sun.nio.ch
>> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>>         at java.base@16.0.2/sun.nio.ch
>> .SelectorImpl.select(SelectorImpl.java:146)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector.select(ManagedSelector.java:190)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
>> Source)
>>         at
>> app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
>>         at
>> app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>    9) Thread[id=428, name=zkCallback-32-thread-4, state=TIMED_WAITING,
>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>         at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
>>         at java.base@16.0.2
>> /java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
>>         at java.base@16.0.2
>> /java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
>>         at java.base@16.0.2
>> /java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
>>         at java.base@16.0.2
>> /java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
>>         at java.base@16.0.2
>> /java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
>>         at java.base@16.0.2
>> /java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
>>         at java.base@16.0.2
>> /java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>   10) Thread[id=418, name=zkCallback-32-thread-2, state=TIMED_WAITING,
>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>         at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
>>         at java.base@16.0.2
>> /java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
>>         at java.base@16.0.2
>> /java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
>>         at java.base@16.0.2
>> /java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
>>         at java.base@16.0.2
>> /java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
>>         at java.base@16.0.2
>> /java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
>>         at java.base@16.0.2
>> /java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
>>         at java.base@16.0.2
>> /java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>   11) Thread[id=365, name=qtp577348919-365, state=RUNNABLE,
>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>>         at java.base@16.0.2/sun.nio.ch
>> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>>         at java.base@16.0.2/sun.nio.ch
>> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>>         at java.base@16.0.2/sun.nio.ch
>> .SelectorImpl.select(SelectorImpl.java:146)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector.select(ManagedSelector.java:190)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
>> Source)
>>         at
>> app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
>>         at
>> app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>   12) Thread[id=369, name=qtp1790869424-369, state=RUNNABLE,
>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>>         at java.base@16.0.2/sun.nio.ch
>> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>>         at java.base@16.0.2/sun.nio.ch
>> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>>         at java.base@16.0.2/sun.nio.ch
>> .SelectorImpl.select(SelectorImpl.java:146)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector.select(ManagedSelector.java:190)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
>> Source)
>>         at
>> app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
>>         at
>> app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>   13) Thread[id=387, name=httpShardExecutor-26-thread-1, state=RUNNABLE,
>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>>         at java.base@16.0.2/sun.nio.ch
>> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>>         at java.base@16.0.2/sun.nio.ch
>> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>>         at java.base@16.0.2/sun.nio.ch
>> .SelectorImpl.select(SelectorImpl.java:146)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector.select(ManagedSelector.java:190)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
>> Source)
>>         at
>> app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:271)
>>         at
>> app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$349/0x0000000801384860.run(Unknown
>> Source)
>>         at java.base@16.0.2
>> /java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
>>         at java.base@16.0.2
>> /java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>   14) Thread[id=355, name=qtp577348919-355, state=RUNNABLE,
>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>>         at java.base@16.0.2/sun.nio.ch
>> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>>         at java.base@16.0.2/sun.nio.ch
>> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>>         at java.base@16.0.2/sun.nio.ch
>> .SelectorImpl.select(SelectorImpl.java:146)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector.select(ManagedSelector.java:190)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
>> Source)
>>         at
>> app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
>>         at
>> app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>   15) Thread[id=396, name=zkConnectionManagerCallback-33-thread-1,
>> state=WAITING, group=TGRP-HdfsCloudIncrementalBackupTest]
>>         at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
>>         at java.base@16.0.2
>> /java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
>>         at java.base@16.0.2
>> /java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
>>         at java.base@16.0.2
>> /java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
>>         at java.base@16.0.2
>> /java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
>>         at java.base@16.0.2
>> /java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
>>         at java.base@16.0.2
>> /java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1056)
>>         at java.base@16.0.2
>> /java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
>>         at java.base@16.0.2
>> /java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>   16) Thread[id=362, name=qtp577348919-362, state=RUNNABLE,
>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>>         at java.base@16.0.2/sun.nio.ch
>> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>>         at java.base@16.0.2/sun.nio.ch
>> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>>         at java.base@16.0.2/sun.nio.ch
>> .SelectorImpl.select(SelectorImpl.java:146)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector.select(ManagedSelector.java:190)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
>> Source)
>>         at
>> app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
>>         at
>> app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>   17) Thread[id=363, name=qtp577348919-363, state=RUNNABLE,
>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>>         at java.base@16.0.2/sun.nio.ch
>> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>>         at java.base@16.0.2/sun.nio.ch
>> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>>         at java.base@16.0.2/sun.nio.ch
>> .SelectorImpl.select(SelectorImpl.java:146)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector.select(ManagedSelector.java:190)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
>> Source)
>>         at
>> app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
>>         at
>> app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>   18) Thread[id=389, name=jetty-launcher-9-thread-2-EventThread,
>> state=WAITING, group=TGRP-HdfsCloudIncrementalBackupTest]
>>         at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
>>         at java.base@16.0.2
>> /java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
>>         at java.base@16.0.2
>> /java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:505)
>>         at java.base@16.0.2
>> /java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137)
>>         at java.base@16.0.2
>> /java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1614)
>>         at java.base@16.0.2
>> /java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
>>         at
>> app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:549)
>>   19) Thread[id=374,
>> name=qtp1790869424-374-acceptor-0@31e11e51-ServerConnector@442851d5{ssl,
>> (ssl, alpn, http/1.1, h2)}{127.0.0.1:44953}, state=RUNNABLE,
>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>         at java.base@16.0.2/sun.nio.ch.Net.accept(Native Method)
>>         at java.base@16.0.2/sun.nio.ch
>> .ServerSocketChannelImpl.implAccept(ServerSocketChannelImpl.java:424)
>>         at java.base@16.0.2/sun.nio.ch
>> .ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:390)
>>         at
>> app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388)
>>         at
>> app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:704)
>>         at
>> app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
>>         at
>> app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>   20) Thread[id=356, name=qtp1790869424-356, state=RUNNABLE,
>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>>         at java.base@16.0.2/sun.nio.ch
>> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>>         at java.base@16.0.2/sun.nio.ch
>> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>>         at java.base@16.0.2/sun.nio.ch
>> .SelectorImpl.select(SelectorImpl.java:146)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector.select(ManagedSelector.java:190)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
>> Source)
>>         at
>> app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
>>         at
>> app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>   21) Thread[id=417, name=zkCallback-27-thread-2, state=TIMED_WAITING,
>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>         at java.base@16.0.2/jdk.internal.misc.Unsafe.park(Native Method)
>>         at java.base@16.0.2
>> /java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
>>         at java.base@16.0.2
>> /java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
>>         at java.base@16.0.2
>> /java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
>>         at java.base@16.0.2
>> /java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
>>         at java.base@16.0.2
>> /java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1055)
>>         at java.base@16.0.2
>> /java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1116)
>>         at java.base@16.0.2
>> /java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>   22) Thread[id=359, name=qtp577348919-359, state=RUNNABLE,
>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>>         at java.base@16.0.2/sun.nio.ch
>> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>>         at java.base@16.0.2/sun.nio.ch
>> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>>         at java.base@16.0.2/sun.nio.ch
>> .SelectorImpl.select(SelectorImpl.java:146)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector.select(ManagedSelector.java:190)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
>> Source)
>>         at
>> app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
>>         at
>> app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>   23) Thread[id=386, name=h2sc-23-thread-1, state=RUNNABLE,
>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>>         at java.base@16.0.2/sun.nio.ch
>> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>>         at java.base@16.0.2/sun.nio.ch
>> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>>         at java.base@16.0.2/sun.nio.ch
>> .SelectorImpl.select(SelectorImpl.java:146)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector.select(ManagedSelector.java:190)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
>> Source)
>>         at
>> app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:271)
>>         at
>> app//org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$349/0x0000000801384860.run(Unknown
>> Source)
>>         at java.base@16.0.2
>> /java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
>>         at java.base@16.0.2
>> /java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:630)
>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>   24) Thread[id=366, name=qtp1790869424-366, state=RUNNABLE,
>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>>         at java.base@16.0.2/sun.nio.ch
>> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>>         at java.base@16.0.2/sun.nio.ch
>> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>>         at java.base@16.0.2/sun.nio.ch
>> .SelectorImpl.select(SelectorImpl.java:146)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector.select(ManagedSelector.java:190)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
>> Source)
>>         at
>> app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
>>         at
>> app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>   25) Thread[id=402, name=CloserThread, state=WAITING,
>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>         at java.base@16.0.2/java.lang.Object.wait(Native Method)
>>         at java.base@16.0.2/java.lang.Object.wait(Object.java:320)
>>         at
>> app//org.apache.solr.core.CloserThread.run(CoreContainer.java:2506)
>>   26) Thread[id=361, name=qtp1790869424-361, state=RUNNABLE,
>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>>         at java.base@16.0.2/sun.nio.ch
>> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>>         at java.base@16.0.2/sun.nio.ch
>> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>>         at java.base@16.0.2/sun.nio.ch
>> .SelectorImpl.select(SelectorImpl.java:146)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector.nioSelect(ManagedSelector.java:183)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector.select(ManagedSelector.java:190)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector$SelectorProducer.select(ManagedSelector.java:606)
>>         at app//org.eclipse.jetty.io
>> .ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
>>         at
>> app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137)
>>         at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$379/0x00000008013ccf88.run(Unknown
>> Source)
>>         at
>> app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
>>         at
>> app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
>>         at java.base@16.0.2/java.lang.Thread.run(Thread.java:831)
>>   27) Thread[id=393, name=h2sc-30-thread-1, state=RUNNABLE,
>> group=TGRP-HdfsCloudIncrementalBackupTest]
>>         at java.base@16.0.2/sun.nio.ch.EPoll.wait(Native Method)
>>         at java.base@16.0.2/sun.nio.ch
>> .EPollSelectorImpl.doSelect(EPollSelectorImpl.java:120)
>>         at java.base@16.0.2/sun.nio.ch
>> .SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
>>
>