You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2018/10/27 11:18:25 UTC

[JENKINS] Lucene-Solr-7.x-Linux (64bit/jdk-11) - Build # 2986 - Unstable!

Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Linux/2986/
Java: 64bit/jdk-11 -XX:-UseCompressedOops -XX:+UseG1GC

55 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.TestCloudRecovery

Error Message:
Could not find collection:collection1

Stack Trace:
java.lang.AssertionError: Could not find collection:collection1
	at __randomizedtesting.SeedInfo.seed([98E8934807BC269A]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNotNull(Assert.java:526)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:155)
	at org.apache.solr.cloud.TestCloudRecovery.setupCluster(TestCloudRecovery.java:70)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1742)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:875)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.base/java.lang.Thread.run(Thread.java:834)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.TestCloudRecovery

Error Message:
Could not find collection:collection1

Stack Trace:
java.lang.AssertionError: Could not find collection:collection1
	at __randomizedtesting.SeedInfo.seed([98E8934807BC269A]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNotNull(Assert.java:526)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:155)
	at org.apache.solr.cloud.TestCloudRecovery.setupCluster(TestCloudRecovery.java:70)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1742)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:875)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.base/java.lang.Thread.run(Thread.java:834)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.TestCloudRecovery

Error Message:
Could not find collection:collection1

Stack Trace:
java.lang.AssertionError: Could not find collection:collection1
	at __randomizedtesting.SeedInfo.seed([98E8934807BC269A]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNotNull(Assert.java:526)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:155)
	at org.apache.solr.cloud.TestCloudRecovery.setupCluster(TestCloudRecovery.java:70)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1742)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:875)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.base/java.lang.Thread.run(Thread.java:834)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.TestCloudRecovery

Error Message:
Could not find collection:collection1

Stack Trace:
java.lang.AssertionError: Could not find collection:collection1
	at __randomizedtesting.SeedInfo.seed([98E8934807BC269A]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNotNull(Assert.java:526)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:155)
	at org.apache.solr.cloud.TestCloudRecovery.setupCluster(TestCloudRecovery.java:70)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1742)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:875)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.base/java.lang.Thread.run(Thread.java:834)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.TestCloudRecovery

Error Message:
Could not find collection:collection1

Stack Trace:
java.lang.AssertionError: Could not find collection:collection1
	at __randomizedtesting.SeedInfo.seed([98E8934807BC269A]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNotNull(Assert.java:526)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:155)
	at org.apache.solr.cloud.TestCloudRecovery.setupCluster(TestCloudRecovery.java:70)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1742)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:875)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.base/java.lang.Thread.run(Thread.java:834)


FAILED:  junit.framework.TestSuite.org.apache.solr.client.solrj.io.stream.StreamDecoratorTest

Error Message:
20 threads leaked from SUITE scope at org.apache.solr.client.solrj.io.stream.StreamDecoratorTest:     1) Thread[id=1477, name=TEST-StreamDecoratorTest.testParallelHavingStream-seed#[543D373DD064BE4E]-SendThread(127.0.0.1:41719), state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/java.lang.Thread.sleep(Native Method)         at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)         at app//org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)         at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)    2) Thread[id=1482, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/java.lang.Thread.sleep(Native Method)         at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.base@11/java.lang.Thread.run(Thread.java:834)    3) Thread[id=274, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/java.lang.Thread.sleep(Native Method)         at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.base@11/java.lang.Thread.run(Thread.java:834)    4) Thread[id=1479, name=zkConnectionManagerCallback-725-thread-1, state=WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)         at java.base@11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)         at java.base@11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)         at java.base@11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)         at java.base@11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)         at java.base@11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)         at java.base@11/java.lang.Thread.run(Thread.java:834)    5) Thread[id=1522, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/java.lang.Thread.sleep(Native Method)         at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.base@11/java.lang.Thread.run(Thread.java:834)    6) Thread[id=1518, name=TEST-StreamDecoratorTest.testParallelExecutorStream-seed#[543D373DD064BE4E]-EventThread, state=WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)         at java.base@11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)         at java.base@11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)         at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502)    7) Thread[id=286, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/java.lang.Thread.sleep(Native Method)         at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.base@11/java.lang.Thread.run(Thread.java:834)    8) Thread[id=1523, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/java.lang.Thread.sleep(Native Method)         at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.base@11/java.lang.Thread.run(Thread.java:834)    9) Thread[id=271, name=zkConnectionManagerCallback-111-thread-1, state=WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)         at java.base@11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)         at java.base@11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)         at java.base@11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)         at java.base@11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)         at java.base@11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)         at java.base@11/java.lang.Thread.run(Thread.java:834)   10) Thread[id=1517, name=TEST-StreamDecoratorTest.testParallelExecutorStream-seed#[543D373DD064BE4E]-SendThread(127.0.0.1:41719), state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/java.lang.Thread.sleep(Native Method)         at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)         at app//org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)         at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)   11) Thread[id=270, name=TEST-StreamDecoratorTest.testExecutorStream-seed#[543D373DD064BE4E]-EventThread, state=WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)         at java.base@11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)         at java.base@11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)         at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502)   12) Thread[id=1483, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/java.lang.Thread.sleep(Native Method)         at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.base@11/java.lang.Thread.run(Thread.java:834)   13) Thread[id=269, name=TEST-StreamDecoratorTest.testExecutorStream-seed#[543D373DD064BE4E]-SendThread(127.0.0.1:41719), state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/java.lang.Thread.sleep(Native Method)         at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)         at app//org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)         at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)   14) Thread[id=268, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/java.lang.Thread.sleep(Native Method)         at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.base@11/java.lang.Thread.run(Thread.java:834)   15) Thread[id=283, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/java.lang.Thread.sleep(Native Method)         at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.base@11/java.lang.Thread.run(Thread.java:834)   16) Thread[id=1478, name=TEST-StreamDecoratorTest.testParallelHavingStream-seed#[543D373DD064BE4E]-EventThread, state=WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)         at java.base@11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)         at java.base@11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)         at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502)   17) Thread[id=1519, name=zkConnectionManagerCallback-736-thread-1, state=WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)         at java.base@11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)         at java.base@11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)         at java.base@11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)         at java.base@11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)         at java.base@11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)         at java.base@11/java.lang.Thread.run(Thread.java:834)   18) Thread[id=1516, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/java.lang.Thread.sleep(Native Method)         at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.base@11/java.lang.Thread.run(Thread.java:834)   19) Thread[id=1476, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/java.lang.Thread.sleep(Native Method)         at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.base@11/java.lang.Thread.run(Thread.java:834)   20) Thread[id=275, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/java.lang.Thread.sleep(Native Method)         at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.base@11/java.lang.Thread.run(Thread.java:834)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 20 threads leaked from SUITE scope at org.apache.solr.client.solrj.io.stream.StreamDecoratorTest: 
   1) Thread[id=1477, name=TEST-StreamDecoratorTest.testParallelHavingStream-seed#[543D373DD064BE4E]-SendThread(127.0.0.1:41719), state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)
   2) Thread[id=1482, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.base@11/java.lang.Thread.run(Thread.java:834)
   3) Thread[id=274, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.base@11/java.lang.Thread.run(Thread.java:834)
   4) Thread[id=1479, name=zkConnectionManagerCallback-725-thread-1, state=WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11/java.lang.Thread.run(Thread.java:834)
   5) Thread[id=1522, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.base@11/java.lang.Thread.run(Thread.java:834)
   6) Thread[id=1518, name=TEST-StreamDecoratorTest.testParallelExecutorStream-seed#[543D373DD064BE4E]-EventThread, state=WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502)
   7) Thread[id=286, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.base@11/java.lang.Thread.run(Thread.java:834)
   8) Thread[id=1523, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.base@11/java.lang.Thread.run(Thread.java:834)
   9) Thread[id=271, name=zkConnectionManagerCallback-111-thread-1, state=WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11/java.lang.Thread.run(Thread.java:834)
  10) Thread[id=1517, name=TEST-StreamDecoratorTest.testParallelExecutorStream-seed#[543D373DD064BE4E]-SendThread(127.0.0.1:41719), state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)
  11) Thread[id=270, name=TEST-StreamDecoratorTest.testExecutorStream-seed#[543D373DD064BE4E]-EventThread, state=WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502)
  12) Thread[id=1483, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.base@11/java.lang.Thread.run(Thread.java:834)
  13) Thread[id=269, name=TEST-StreamDecoratorTest.testExecutorStream-seed#[543D373DD064BE4E]-SendThread(127.0.0.1:41719), state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)
  14) Thread[id=268, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.base@11/java.lang.Thread.run(Thread.java:834)
  15) Thread[id=283, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.base@11/java.lang.Thread.run(Thread.java:834)
  16) Thread[id=1478, name=TEST-StreamDecoratorTest.testParallelHavingStream-seed#[543D373DD064BE4E]-EventThread, state=WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502)
  17) Thread[id=1519, name=zkConnectionManagerCallback-736-thread-1, state=WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11/java.lang.Thread.run(Thread.java:834)
  18) Thread[id=1516, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.base@11/java.lang.Thread.run(Thread.java:834)
  19) Thread[id=1476, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.base@11/java.lang.Thread.run(Thread.java:834)
  20) Thread[id=275, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.base@11/java.lang.Thread.run(Thread.java:834)
	at __randomizedtesting.SeedInfo.seed([543D373DD064BE4E]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.client.solrj.io.stream.StreamDecoratorTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=1477, name=TEST-StreamDecoratorTest.testParallelHavingStream-seed#[543D373DD064BE4E]-SendThread(127.0.0.1:41719), state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/java.lang.Thread.sleep(Native Method)         at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)         at app//org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)         at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)    2) Thread[id=1479, name=zkConnectionManagerCallback-725-thread-1, state=WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)         at java.base@11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)         at java.base@11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)         at java.base@11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)         at java.base@11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)         at java.base@11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)         at java.base@11/java.lang.Thread.run(Thread.java:834)    3) Thread[id=271, name=zkConnectionManagerCallback-111-thread-1, state=WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)         at java.base@11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)         at java.base@11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)         at java.base@11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)         at java.base@11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)         at java.base@11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)         at java.base@11/java.lang.Thread.run(Thread.java:834)    4) Thread[id=1517, name=TEST-StreamDecoratorTest.testParallelExecutorStream-seed#[543D373DD064BE4E]-SendThread(127.0.0.1:41719), state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/java.lang.Thread.sleep(Native Method)         at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1054)    5) Thread[id=269, name=TEST-StreamDecoratorTest.testExecutorStream-seed#[543D373DD064BE4E]-SendThread(127.0.0.1:41719), state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/java.lang.Thread.sleep(Native Method)         at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)         at app//org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)         at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)    6) Thread[id=1519, name=zkConnectionManagerCallback-736-thread-1, state=WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)         at java.base@11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)         at java.base@11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)         at java.base@11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)         at java.base@11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)         at java.base@11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)         at java.base@11/java.lang.Thread.run(Thread.java:834)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=1477, name=TEST-StreamDecoratorTest.testParallelHavingStream-seed#[543D373DD064BE4E]-SendThread(127.0.0.1:41719), state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)
   2) Thread[id=1479, name=zkConnectionManagerCallback-725-thread-1, state=WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11/java.lang.Thread.run(Thread.java:834)
   3) Thread[id=271, name=zkConnectionManagerCallback-111-thread-1, state=WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11/java.lang.Thread.run(Thread.java:834)
   4) Thread[id=1517, name=TEST-StreamDecoratorTest.testParallelExecutorStream-seed#[543D373DD064BE4E]-SendThread(127.0.0.1:41719), state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1054)
   5) Thread[id=269, name=TEST-StreamDecoratorTest.testExecutorStream-seed#[543D373DD064BE4E]-SendThread(127.0.0.1:41719), state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)
   6) Thread[id=1519, name=zkConnectionManagerCallback-736-thread-1, state=WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11/java.lang.Thread.run(Thread.java:834)
	at __randomizedtesting.SeedInfo.seed([543D373DD064BE4E]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.client.solrj.io.stream.StreamDecoratorTest

Error Message:
20 threads leaked from SUITE scope at org.apache.solr.client.solrj.io.stream.StreamDecoratorTest:     1) Thread[id=278, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/java.lang.Thread.sleep(Native Method)         at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.base@11/java.lang.Thread.run(Thread.java:834)    2) Thread[id=272, name=TEST-StreamDecoratorTest.testExecutorStream-seed#[543D373DD064BE4E]-SendThread(127.0.0.1:45557), state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/java.lang.Thread.sleep(Native Method)         at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)         at app//org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)         at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)    3) Thread[id=1522, name=TEST-StreamDecoratorTest.testParallelExecutorStream-seed#[543D373DD064BE4E]-SendThread(127.0.0.1:45557), state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/java.lang.Thread.sleep(Native Method)         at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1054)    4) Thread[id=1528, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/java.lang.Thread.sleep(Native Method)         at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.base@11/java.lang.Thread.run(Thread.java:834)    5) Thread[id=1521, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/java.lang.Thread.sleep(Native Method)         at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.base@11/java.lang.Thread.run(Thread.java:834)    6) Thread[id=1523, name=TEST-StreamDecoratorTest.testParallelExecutorStream-seed#[543D373DD064BE4E]-EventThread, state=WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)         at java.base@11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)         at java.base@11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)         at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502)    7) Thread[id=273, name=TEST-StreamDecoratorTest.testExecutorStream-seed#[543D373DD064BE4E]-EventThread, state=WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)         at java.base@11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)         at java.base@11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)         at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502)    8) Thread[id=1478, name=TEST-StreamDecoratorTest.testParallelHavingStream-seed#[543D373DD064BE4E]-SendThread(127.0.0.1:45557), state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/java.lang.Thread.sleep(Native Method)         at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)         at app//org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)         at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)    9) Thread[id=1483, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/java.lang.Thread.sleep(Native Method)         at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.base@11/java.lang.Thread.run(Thread.java:834)   10) Thread[id=274, name=zkConnectionManagerCallback-111-thread-1, state=WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)         at java.base@11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)         at java.base@11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)         at java.base@11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)         at java.base@11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)         at java.base@11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)         at java.base@11/java.lang.Thread.run(Thread.java:834)   11) Thread[id=277, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/java.lang.Thread.sleep(Native Method)         at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.base@11/java.lang.Thread.run(Thread.java:834)   12) Thread[id=1524, name=zkConnectionManagerCallback-739-thread-1, state=WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)         at java.base@11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)         at java.base@11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)         at java.base@11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)         at java.base@11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)         at java.base@11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)         at java.base@11/java.lang.Thread.run(Thread.java:834)   13) Thread[id=289, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/java.lang.Thread.sleep(Native Method)         at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.base@11/java.lang.Thread.run(Thread.java:834)   14) Thread[id=284, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/java.lang.Thread.sleep(Native Method)         at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.base@11/java.lang.Thread.run(Thread.java:834)   15) Thread[id=1480, name=zkConnectionManagerCallback-725-thread-1, state=WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)         at java.base@11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)         at java.base@11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)         at java.base@11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)         at java.base@11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)         at java.base@11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)         at java.base@11/java.lang.Thread.run(Thread.java:834)   16) Thread[id=1484, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/java.lang.Thread.sleep(Native Method)         at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.base@11/java.lang.Thread.run(Thread.java:834)   17) Thread[id=1527, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/java.lang.Thread.sleep(Native Method)         at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.base@11/java.lang.Thread.run(Thread.java:834)   18) Thread[id=1479, name=TEST-StreamDecoratorTest.testParallelHavingStream-seed#[543D373DD064BE4E]-EventThread, state=WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)         at java.base@11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)         at java.base@11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)         at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502)   19) Thread[id=271, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/java.lang.Thread.sleep(Native Method)         at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.base@11/java.lang.Thread.run(Thread.java:834)   20) Thread[id=1477, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/java.lang.Thread.sleep(Native Method)         at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.base@11/java.lang.Thread.run(Thread.java:834)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 20 threads leaked from SUITE scope at org.apache.solr.client.solrj.io.stream.StreamDecoratorTest: 
   1) Thread[id=278, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.base@11/java.lang.Thread.run(Thread.java:834)
   2) Thread[id=272, name=TEST-StreamDecoratorTest.testExecutorStream-seed#[543D373DD064BE4E]-SendThread(127.0.0.1:45557), state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)
   3) Thread[id=1522, name=TEST-StreamDecoratorTest.testParallelExecutorStream-seed#[543D373DD064BE4E]-SendThread(127.0.0.1:45557), state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1054)
   4) Thread[id=1528, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.base@11/java.lang.Thread.run(Thread.java:834)
   5) Thread[id=1521, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.base@11/java.lang.Thread.run(Thread.java:834)
   6) Thread[id=1523, name=TEST-StreamDecoratorTest.testParallelExecutorStream-seed#[543D373DD064BE4E]-EventThread, state=WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502)
   7) Thread[id=273, name=TEST-StreamDecoratorTest.testExecutorStream-seed#[543D373DD064BE4E]-EventThread, state=WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502)
   8) Thread[id=1478, name=TEST-StreamDecoratorTest.testParallelHavingStream-seed#[543D373DD064BE4E]-SendThread(127.0.0.1:45557), state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)
   9) Thread[id=1483, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.base@11/java.lang.Thread.run(Thread.java:834)
  10) Thread[id=274, name=zkConnectionManagerCallback-111-thread-1, state=WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11/java.lang.Thread.run(Thread.java:834)
  11) Thread[id=277, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.base@11/java.lang.Thread.run(Thread.java:834)
  12) Thread[id=1524, name=zkConnectionManagerCallback-739-thread-1, state=WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11/java.lang.Thread.run(Thread.java:834)
  13) Thread[id=289, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.base@11/java.lang.Thread.run(Thread.java:834)
  14) Thread[id=284, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.base@11/java.lang.Thread.run(Thread.java:834)
  15) Thread[id=1480, name=zkConnectionManagerCallback-725-thread-1, state=WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11/java.lang.Thread.run(Thread.java:834)
  16) Thread[id=1484, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.base@11/java.lang.Thread.run(Thread.java:834)
  17) Thread[id=1527, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.base@11/java.lang.Thread.run(Thread.java:834)
  18) Thread[id=1479, name=TEST-StreamDecoratorTest.testParallelHavingStream-seed#[543D373DD064BE4E]-EventThread, state=WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502)
  19) Thread[id=271, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.base@11/java.lang.Thread.run(Thread.java:834)
  20) Thread[id=1477, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.base@11/java.lang.Thread.run(Thread.java:834)
	at __randomizedtesting.SeedInfo.seed([543D373DD064BE4E]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.client.solrj.io.stream.StreamDecoratorTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=272, name=TEST-StreamDecoratorTest.testExecutorStream-seed#[543D373DD064BE4E]-SendThread(127.0.0.1:45557), state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/java.lang.Thread.sleep(Native Method)         at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)         at app//org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)         at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)    2) Thread[id=274, name=zkConnectionManagerCallback-111-thread-1, state=WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)         at java.base@11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)         at java.base@11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)         at java.base@11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)         at java.base@11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)         at java.base@11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)         at java.base@11/java.lang.Thread.run(Thread.java:834)    3) Thread[id=1522, name=TEST-StreamDecoratorTest.testParallelExecutorStream-seed#[543D373DD064BE4E]-SendThread(127.0.0.1:45557), state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/java.lang.Thread.sleep(Native Method)         at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)         at app//org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)         at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)    4) Thread[id=1524, name=zkConnectionManagerCallback-739-thread-1, state=WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)         at java.base@11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)         at java.base@11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)         at java.base@11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)         at java.base@11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)         at java.base@11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)         at java.base@11/java.lang.Thread.run(Thread.java:834)    5) Thread[id=1480, name=zkConnectionManagerCallback-725-thread-1, state=WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)         at java.base@11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)         at java.base@11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)         at java.base@11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)         at java.base@11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)         at java.base@11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)         at java.base@11/java.lang.Thread.run(Thread.java:834)    6) Thread[id=1478, name=TEST-StreamDecoratorTest.testParallelHavingStream-seed#[543D373DD064BE4E]-SendThread(127.0.0.1:45557), state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/java.lang.Thread.sleep(Native Method)         at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)         at app//org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)         at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=272, name=TEST-StreamDecoratorTest.testExecutorStream-seed#[543D373DD064BE4E]-SendThread(127.0.0.1:45557), state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)
   2) Thread[id=274, name=zkConnectionManagerCallback-111-thread-1, state=WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11/java.lang.Thread.run(Thread.java:834)
   3) Thread[id=1522, name=TEST-StreamDecoratorTest.testParallelExecutorStream-seed#[543D373DD064BE4E]-SendThread(127.0.0.1:45557), state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)
   4) Thread[id=1524, name=zkConnectionManagerCallback-739-thread-1, state=WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11/java.lang.Thread.run(Thread.java:834)
   5) Thread[id=1480, name=zkConnectionManagerCallback-725-thread-1, state=WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11/java.lang.Thread.run(Thread.java:834)
   6) Thread[id=1478, name=TEST-StreamDecoratorTest.testParallelHavingStream-seed#[543D373DD064BE4E]-SendThread(127.0.0.1:45557), state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)
	at __randomizedtesting.SeedInfo.seed([543D373DD064BE4E]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.client.solrj.io.stream.StreamDecoratorTest

Error Message:
20 threads leaked from SUITE scope at org.apache.solr.client.solrj.io.stream.StreamDecoratorTest:     1) Thread[id=272, name=zkConnectionManagerCallback-111-thread-1, state=WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)         at java.base@11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)         at java.base@11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)         at java.base@11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)         at java.base@11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)         at java.base@11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)         at java.base@11/java.lang.Thread.run(Thread.java:834)    2) Thread[id=288, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/java.lang.Thread.sleep(Native Method)         at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.base@11/java.lang.Thread.run(Thread.java:834)    3) Thread[id=1451, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/java.lang.Thread.sleep(Native Method)         at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.base@11/java.lang.Thread.run(Thread.java:834)    4) Thread[id=1453, name=TEST-StreamDecoratorTest.testParallelHavingStream-seed#[543D373DD064BE4E]-EventThread, state=WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)         at java.base@11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)         at java.base@11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)         at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502)    5) Thread[id=1513, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/java.lang.Thread.sleep(Native Method)         at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.base@11/java.lang.Thread.run(Thread.java:834)    6) Thread[id=1514, name=TEST-StreamDecoratorTest.testParallelExecutorStream-seed#[543D373DD064BE4E]-SendThread(127.0.0.1:38353), state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/java.lang.Thread.sleep(Native Method)         at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)         at app//org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)         at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)    7) Thread[id=284, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/java.lang.Thread.sleep(Native Method)         at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.base@11/java.lang.Thread.run(Thread.java:834)    8) Thread[id=1452, name=TEST-StreamDecoratorTest.testParallelHavingStream-seed#[543D373DD064BE4E]-SendThread(127.0.0.1:38353), state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/java.lang.Thread.sleep(Native Method)         at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1054)    9) Thread[id=1454, name=zkConnectionManagerCallback-713-thread-1, state=WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)         at java.base@11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)         at java.base@11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)         at java.base@11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)         at java.base@11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)         at java.base@11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)         at java.base@11/java.lang.Thread.run(Thread.java:834)   10) Thread[id=1458, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/java.lang.Thread.sleep(Native Method)         at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.base@11/java.lang.Thread.run(Thread.java:834)   11) Thread[id=1519, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/java.lang.Thread.sleep(Native Method)         at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.base@11/java.lang.Thread.run(Thread.java:834)   12) Thread[id=1515, name=TEST-StreamDecoratorTest.testParallelExecutorStream-seed#[543D373DD064BE4E]-EventThread, state=WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)         at java.base@11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)         at java.base@11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)         at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502)   13) Thread[id=1520, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/java.lang.Thread.sleep(Native Method)         at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.base@11/java.lang.Thread.run(Thread.java:834)   14) Thread[id=271, name=TEST-StreamDecoratorTest.testExecutorStream-seed#[543D373DD064BE4E]-EventThread, state=WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)         at java.base@11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)         at java.base@11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)         at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502)   15) Thread[id=270, name=TEST-StreamDecoratorTest.testExecutorStream-seed#[543D373DD064BE4E]-SendThread(127.0.0.1:38353), state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/java.lang.Thread.sleep(Native Method)         at app//org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:230)         at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1249)         at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1173)   16) Thread[id=269, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/java.lang.Thread.sleep(Native Method)         at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.base@11/java.lang.Thread.run(Thread.java:834)   17) Thread[id=276, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/java.lang.Thread.sleep(Native Method)         at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.base@11/java.lang.Thread.run(Thread.java:834)   18) Thread[id=1457, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/java.lang.Thread.sleep(Native Method)         at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.base@11/java.lang.Thread.run(Thread.java:834)   19) Thread[id=1516, name=zkConnectionManagerCallback-736-thread-1, state=WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)         at java.base@11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)         at java.base@11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)         at java.base@11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)         at java.base@11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)         at java.base@11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)         at java.base@11/java.lang.Thread.run(Thread.java:834)   20) Thread[id=275, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]         at java.base@11/java.lang.Thread.sleep(Native Method)         at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.base@11/java.lang.Thread.run(Thread.java:834)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 20 threads leaked from SUITE scope at org.apache.solr.client.solrj.io.stream.StreamDecoratorTest: 
   1) Thread[id=272, name=zkConnectionManagerCallback-111-thread-1, state=WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11/java.lang.Thread.run(Thread.java:834)
   2) Thread[id=288, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.base@11/java.lang.Thread.run(Thread.java:834)
   3) Thread[id=1451, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.base@11/java.lang.Thread.run(Thread.java:834)
   4) Thread[id=1453, name=TEST-StreamDecoratorTest.testParallelHavingStream-seed#[543D373DD064BE4E]-EventThread, state=WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502)
   5) Thread[id=1513, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.base@11/java.lang.Thread.run(Thread.java:834)
   6) Thread[id=1514, name=TEST-StreamDecoratorTest.testParallelExecutorStream-seed#[543D373DD064BE4E]-SendThread(127.0.0.1:38353), state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)
   7) Thread[id=284, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.base@11/java.lang.Thread.run(Thread.java:834)
   8) Thread[id=1452, name=TEST-StreamDecoratorTest.testParallelHavingStream-seed#[543D373DD064BE4E]-SendThread(127.0.0.1:38353), state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1054)
   9) Thread[id=1454, name=zkConnectionManagerCallback-713-thread-1, state=WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.base@11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at java.base@11/java.lang.Thread.run(Thread.java:834)
  10) Thread[id=1458, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.base@11/java.lang.Thread.run(Thread.java:834)
  11) Thread[id=1519, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.base@11/java.lang.Thread.run(Thread.java:834)
  12) Thread[id=1515, name=TEST-StreamDecoratorTest.testParallelExecutorStream-seed#[543D373DD064BE4E]-EventThread, state=WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502)
  13) Thread[id=1520, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.base@11/java.lang.Thread.run(Thread.java:834)
  14) Thread[id=271, name=TEST-StreamDecoratorTest.testExecutorStream-seed#[543D373DD064BE4E]-EventThread, state=WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502)
  15) Thread[id=270, name=TEST-StreamDecoratorTest.testExecutorStream-seed#[543D373DD064BE4E]-SendThread(127.0.0.1:38353), state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:230)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1249)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1173)
  16) Thread[id=269, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.base@11/java.lang.Thread.run(Thread.java:834)
  17) Thread[id=276, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.base@11/java.lang.Thread.run(Thread.java:834)
  18) Thread[id=1457, name=Connection evictor, state=TIMED_WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/java.lang.Thread.sleep(Native Method)
        at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.base@11/java.lang.Thread.run(Thread.java:834)
  19) Thread[id=1516, name=zkConnectionManagerCallback-736-thread-1, state=WAITING, group=TGRP-StreamDecoratorTest]
        at java.base@11/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2081)
        at java.base@11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:433)
        at java.base@11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1054)
        at java.base@11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1114)
        at java.

[...truncated too long message...]

 [junit4]   2> 1653846 INFO  (jetty-closer-3749-thread-1) [    ] o.a.s.c.ZkController Remove node as live in ZooKeeper:/live_nodes/127.0.0.1:35615_solr
   [junit4]   2> 1653847 INFO  (jetty-closer-3749-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
   [junit4]   2> 1653847 INFO  (zkCallback-3760-thread-2) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
   [junit4]   2> 1653847 INFO  (zkCallback-3766-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
   [junit4]   2> 1653847 INFO  (zkCallback-3780-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
   [junit4]   2> 1653847 INFO  (zkCallback-3775-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
   [junit4]   2> 1653847 INFO  (jetty-closer-3749-thread-2) [    ] o.a.s.c.ZkController Remove node as live in ZooKeeper:/live_nodes/127.0.0.1:46637_solr
   [junit4]   2> 1653848 INFO  (jetty-closer-3749-thread-2) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
   [junit4]   2> 1653848 INFO  (zkCallback-3775-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
   [junit4]   2> 1653848 INFO  (zkCallback-3760-thread-2) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
   [junit4]   2> 1653848 INFO  (jetty-closer-3749-thread-2) [    ] o.a.s.c.Overseer Overseer (id=72058434230222852-127.0.0.1:46637_solr-n_0000000000) closing
   [junit4]   2> 1653848 INFO  (OverseerStateUpdate-72058434230222852-127.0.0.1:46637_solr-n_0000000000) [n:127.0.0.1:46637_solr    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:46637_solr
   [junit4]   2> 1653848 WARN  (OverseerAutoScalingTriggerThread-72058434230222852-127.0.0.1:46637_solr-n_0000000000) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 1653850 INFO  (jetty-closer-3749-thread-2) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@687aa68{/solr,null,UNAVAILABLE}
   [junit4]   2> 1653850 INFO  (jetty-closer-3749-thread-2) [    ] o.e.j.s.session node0 Stopped scavenging
   [junit4]   2> 1655348 WARN  (zkCallback-3766-thread-1) [    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 1655348 INFO  (jetty-closer-3749-thread-1) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@27d7cc9a{/solr,null,UNAVAILABLE}
   [junit4]   2> 1655349 INFO  (jetty-closer-3749-thread-1) [    ] o.e.j.s.session node0 Stopped scavenging
   [junit4]   2> 1655349 ERROR (SUITE-TestCloudRecovery-seed#[98E8934807BC269A]-worker) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 1655350 INFO  (SUITE-TestCloudRecovery-seed#[98E8934807BC269A]-worker) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:34205 34205
   [junit4]   2> 1655477 INFO  (Thread-2174) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:34205 34205
   [junit4]   2> 1655478 WARN  (Thread-2174) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	5	/solr/aliases.json
   [junit4]   2> 	3	/solr/collections/collection1/terms/shard2
   [junit4]   2> 	3	/solr/collections/collection1/terms/shard1
   [junit4]   2> 	2	/solr/security.json
   [junit4]   2> 	2	/solr/configs/config
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	5	/solr/clusterstate.json
   [junit4]   2> 	5	/solr/clusterprops.json
   [junit4]   2> 	2	/solr/collections/collection1/state.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	5	/solr/live_nodes
   [junit4]   2> 	5	/solr/collections
   [junit4]   2> 
   [junit4]   2> 1655478 INFO  (SUITE-TestCloudRecovery-seed#[98E8934807BC269A]-worker) [    ] o.a.s.SolrTestCaseJ4 ------------------------------------------------------- Done waiting for tracked resources to be released
   [junit4]   2> Oct 27, 2018 11:10:12 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=1789, maxMBSortInHeap=5.95497397579583, sim=RandomSimilarity(queryNorm=false): {}, locale=ksb, timezone=Europe/Rome
   [junit4]   2> NOTE: Linux 4.15.0-38-generic amd64/Oracle Corporation 11 (64-bit)/cpus=8,threads=1,free=346834584,total=536870912
   [junit4]   2> NOTE: All tests run in this JVM: [TestMiniSolrCloudClusterSSL, TestMiniSolrCloudClusterSSL, ShardSplitTest, ShardSplitTest, TestMiniSolrCloudClusterSSL, TestCloudRecovery]
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestCloudRecovery -Dtests.seed=98E8934807BC269A -Dtests.multiplier=3 -Dtests.slow=true -Dtests.badapples=true -Dtests.locale=ksb -Dtests.timezone=Europe/Rome -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   0.00s J1 | TestCloudRecovery (suite) <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: Could not find collection:collection1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([98E8934807BC269A]:0)
   [junit4]    > 	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:155)
   [junit4]    > 	at org.apache.solr.cloud.TestCloudRecovery.setupCluster(TestCloudRecovery.java:70)
   [junit4]    > 	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]    > 	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]    > 	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]    > 	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
   [junit4]    > 	at java.base/java.lang.Thread.run(Thread.java:834)
   [junit4] Completed [20/20 (19!)] on J1 in 44.90s, 0 tests, 1 failure <<< FAILURES!
   [junit4] 
   [junit4] 
   [junit4] Tests with failures [seed: 98E8934807BC269A] (first 10 out of 29):
   [junit4]   - org.apache.solr.cloud.TestMiniSolrCloudClusterSSL.testSslWithCheckPeerName
   [junit4]   - org.apache.solr.cloud.TestMiniSolrCloudClusterSSL.testSslAndNoClientAuth
   [junit4]   - org.apache.solr.cloud.TestMiniSolrCloudClusterSSL.testSslAndNoClientAuth
   [junit4]   - org.apache.solr.cloud.TestMiniSolrCloudClusterSSL.testSslWithInvalidPeerName
   [junit4]   - org.apache.solr.cloud.TestMiniSolrCloudClusterSSL.testSslWithCheckPeerName
   [junit4]   - org.apache.solr.cloud.TestMiniSolrCloudClusterSSL.testSslWithInvalidPeerName
   [junit4]   - org.apache.solr.cloud.TestMiniSolrCloudClusterSSL.testSslWithCheckPeerName
   [junit4]   - org.apache.solr.metrics.reporters.solr.SolrCloudReportersTest.testDefaultPlugins
   [junit4]   - org.apache.solr.metrics.reporters.solr.SolrCloudReportersTest.testExplicitConfiguration
   [junit4]   - org.apache.solr.cloud.TestCloudRecovery (suite)
   [junit4] 
   [junit4] 
   [junit4] JVM J0:     0.71 ..  1620.45 =  1619.74s
   [junit4] JVM J1:     0.73 ..  1656.56 =  1655.82s
   [junit4] JVM J2:     0.71 ..  1617.33 =  1616.61s
   [junit4] Execution time total: 27 minutes 36 seconds
   [junit4] Tests summary: 20 suites, 95 tests, 5 suite-level errors, 7 errors, 17 failures

BUILD FAILED
/home/jenkins/workspace/Lucene-Solr-7.x-Linux/lucene/common-build.xml:1567: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-7.x-Linux/lucene/common-build.xml:1092: There were test failures: 20 suites, 95 tests, 5 suite-level errors, 7 errors, 17 failures [seed: 98E8934807BC269A]

Total time: 27 minutes 38 seconds

[repro] Setting last failure code to 256

[repro] Failures:
[repro]   3/5 failed: org.apache.solr.client.solrj.io.stream.StreamDecoratorTest
[repro]   4/5 failed: org.apache.solr.metrics.reporters.solr.SolrCloudReportersTest
[repro]   5/5 failed: org.apache.solr.cloud.TestCloudRecovery
[repro]   5/5 failed: org.apache.solr.cloud.TestMiniSolrCloudClusterSSL
[repro]   5/5 failed: org.apache.solr.cloud.api.collections.ShardSplitTest
[repro] Exiting with code 256
+ mv lucene/build lucene/build.repro
+ mv solr/build solr/build.repro
+ mv lucene/build.orig lucene/build
+ mv solr/build.orig solr/build
Archiving artifacts
Setting ANT_1_8_2_HOME=/var/lib/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
[WARNINGS] Parsing warnings in console log with parser Java Compiler (javac)
Setting ANT_1_8_2_HOME=/var/lib/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
Setting ANT_1_8_2_HOME=/var/lib/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
<Git Blamer> Using GitBlamer to create author and commit information for all warnings.
<Git Blamer> GIT_COMMIT=575d3dac0ff12bc66e4cd7a211661307d43f3348, workspace=/var/lib/jenkins/workspace/Lucene-Solr-7.x-Linux
[WARNINGS] Computing warning deltas based on reference build #2985
Recording test results
Setting ANT_1_8_2_HOME=/var/lib/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)
Setting ANT_1_8_2_HOME=/var/lib/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
Setting ANT_1_8_2_HOME=/var/lib/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
Setting ANT_1_8_2_HOME=/var/lib/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
Setting ANT_1_8_2_HOME=/var/lib/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2

[JENKINS] Lucene-Solr-7.x-Linux (32bit/jdk1.8.0_172) - Build # 2987 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Linux/2987/
Java: 32bit/jdk1.8.0_172 -server -XX:+UseParallelGC

7 tests failed.
FAILED:  org.apache.solr.client.solrj.impl.CloudSolrClientTest.testVersionsAreReturned

Error Message:
Error from server at https://127.0.0.1:42789/solr/collection1_shard2_replica_n2: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=utf-8"/> <title>Error 404 Can not find: /solr/collection1_shard2_replica_n2/update</title> </head> <body><h2>HTTP ERROR 404</h2> <p>Problem accessing /solr/collection1_shard2_replica_n2/update. Reason: <pre>    Can not find: /solr/collection1_shard2_replica_n2/update</pre></p><hr><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.4.11.v20180605</a><hr/>  </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.CloudSolrClient$RouteException: Error from server at https://127.0.0.1:42789/solr/collection1_shard2_replica_n2: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=utf-8"/>
<title>Error 404 Can not find: /solr/collection1_shard2_replica_n2/update</title>
</head>
<body><h2>HTTP ERROR 404</h2>
<p>Problem accessing /solr/collection1_shard2_replica_n2/update. Reason:
<pre>    Can not find: /solr/collection1_shard2_replica_n2/update</pre></p><hr><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.4.11.v20180605</a><hr/>

</body>
</html>

	at __randomizedtesting.SeedInfo.seed([8ACF741D98C8E053:72098D280BD1789B]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:551)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1016)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:884)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:949)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:949)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:949)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:949)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:949)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:817)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
	at org.apache.solr.client.solrj.request.UpdateRequest.commit(UpdateRequest.java:237)
	at org.apache.solr.client.solrj.impl.CloudSolrClientTest.testVersionsAreReturned(CloudSolrClientTest.java:725)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1742)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:935)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:971)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:985)
	at org.junit.rules.ExpectedException$ExpectedExceptionStatement.evaluate(ExpectedException.java:110)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:944)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:830)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:42789/solr/collection1_shard2_replica_n2: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=utf-8"/>
<title>Error 404 Can not find: /solr/collection1_shard2_replica_n2/update</title>
</head>
<body><h2>HTTP ERROR 404</h2>
<p>Problem accessing /solr/collection1_shard2_replica_n2/update. Reason:
<pre>    Can not find: /solr/collection1_shard2_replica_n2/update</pre></p><hr><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.4.11.v20180605</a><hr/>

</body>
</html>

	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:607)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.lambda$directUpdate$0(CloudSolrClient.java:528)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	... 1 more


FAILED:  org.apache.solr.cloud.DeleteReplicaTest.raceConditionOnDeleteAndRegisterReplica

Error Message:
Expected new active leader null Live Nodes: [127.0.0.1:34053_solr, 127.0.0.1:34371_solr, 127.0.0.1:38881_solr] Last available state: DocCollection(raceDeleteReplica_true//collections/raceDeleteReplica_true/state.json/12)={   "pullReplicas":"0",   "replicationFactor":"2",   "shards":{"shard1":{       "range":"80000000-7fffffff",       "state":"active",       "replicas":{         "core_node3":{           "core":"raceDeleteReplica_true_shard1_replica_n1",           "base_url":"https://127.0.0.1:40209/solr",           "node_name":"127.0.0.1:40209_solr",           "state":"down",           "type":"NRT",           "leader":"true"},         "core_node6":{           "core":"raceDeleteReplica_true_shard1_replica_n5",           "base_url":"https://127.0.0.1:40209/solr",           "node_name":"127.0.0.1:40209_solr",           "state":"down",           "type":"NRT"},         "core_node4":{           "core":"raceDeleteReplica_true_shard1_replica_n2",           "base_url":"https://127.0.0.1:34371/solr",           "node_name":"127.0.0.1:34371_solr",           "state":"down",           "type":"NRT"}}}},   "router":{"name":"compositeId"},   "maxShardsPerNode":"1",   "autoAddReplicas":"false",   "nrtReplicas":"2",   "tlogReplicas":"0"}

Stack Trace:
java.lang.AssertionError: Expected new active leader
null
Live Nodes: [127.0.0.1:34053_solr, 127.0.0.1:34371_solr, 127.0.0.1:38881_solr]
Last available state: DocCollection(raceDeleteReplica_true//collections/raceDeleteReplica_true/state.json/12)={
  "pullReplicas":"0",
  "replicationFactor":"2",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node3":{
          "core":"raceDeleteReplica_true_shard1_replica_n1",
          "base_url":"https://127.0.0.1:40209/solr",
          "node_name":"127.0.0.1:40209_solr",
          "state":"down",
          "type":"NRT",
          "leader":"true"},
        "core_node6":{
          "core":"raceDeleteReplica_true_shard1_replica_n5",
          "base_url":"https://127.0.0.1:40209/solr",
          "node_name":"127.0.0.1:40209_solr",
          "state":"down",
          "type":"NRT"},
        "core_node4":{
          "core":"raceDeleteReplica_true_shard1_replica_n2",
          "base_url":"https://127.0.0.1:34371/solr",
          "node_name":"127.0.0.1:34371_solr",
          "state":"down",
          "type":"NRT"}}}},
  "router":{"name":"compositeId"},
  "maxShardsPerNode":"1",
  "autoAddReplicas":"false",
  "nrtReplicas":"2",
  "tlogReplicas":"0"}
	at __randomizedtesting.SeedInfo.seed([EF573F4625E2C3F9:85415E964D108933]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.SolrCloudTestCase.waitForState(SolrCloudTestCase.java:280)
	at org.apache.solr.cloud.DeleteReplicaTest.raceConditionOnDeleteAndRegisterReplica(DeleteReplicaTest.java:334)
	at org.apache.solr.cloud.DeleteReplicaTest.raceConditionOnDeleteAndRegisterReplica(DeleteReplicaTest.java:229)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1742)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:935)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:971)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:985)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:944)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:830)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.DeleteReplicaTest.deleteLiveReplicaTest

Error Message:
Error from server at https://127.0.0.1:38881/solr: Cannot create collection delLiveColl. Value of maxShardsPerNode is 1, and the number of nodes currently live or live and part of your createNodeSet is 3. This allows a maximum of 3 to be created. Value of numShards is 2, value of nrtReplicas is 2, value of tlogReplicas is 0 and value of pullReplicas is 0. This requires 4 shards to be created (higher than the allowed number)

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:38881/solr: Cannot create collection delLiveColl. Value of maxShardsPerNode is 1, and the number of nodes currently live or live and part of your createNodeSet is 3. This allows a maximum of 3 to be created. Value of numShards is 2, value of nrtReplicas is 2, value of tlogReplicas is 0 and value of pullReplicas is 0. This requires 4 shards to be created (higher than the allowed number)
	at __randomizedtesting.SeedInfo.seed([EF573F4625E2C3F9:42378B4D38DD6B8C]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1107)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:884)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:817)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
	at org.apache.solr.cloud.DeleteReplicaTest.deleteLiveReplicaTest(DeleteReplicaTest.java:75)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1742)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:935)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:971)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:985)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:944)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:830)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.DeleteReplicaTest.raceConditionOnDeleteAndRegisterReplica

Error Message:
Expected new active leader null Live Nodes: [127.0.0.1:36825_solr, 127.0.0.1:41063_solr, 127.0.0.1:43487_solr] Last available state: DocCollection(raceDeleteReplica_true//collections/raceDeleteReplica_true/state.json/11)={   "pullReplicas":"0",   "replicationFactor":"2",   "shards":{"shard1":{       "range":"80000000-7fffffff",       "state":"active",       "replicas":{         "core_node4":{           "core":"raceDeleteReplica_true_shard1_replica_n1",           "base_url":"https://127.0.0.1:44359/solr",           "node_name":"127.0.0.1:44359_solr",           "state":"down",           "type":"NRT",           "leader":"true"},         "core_node6":{           "core":"raceDeleteReplica_true_shard1_replica_n5",           "base_url":"https://127.0.0.1:44359/solr",           "node_name":"127.0.0.1:44359_solr",           "state":"down",           "type":"NRT"},         "core_node3":{           "core":"raceDeleteReplica_true_shard1_replica_n2",           "base_url":"https://127.0.0.1:41063/solr",           "node_name":"127.0.0.1:41063_solr",           "state":"down",           "type":"NRT"}}}},   "router":{"name":"compositeId"},   "maxShardsPerNode":"1",   "autoAddReplicas":"false",   "nrtReplicas":"2",   "tlogReplicas":"0"}

Stack Trace:
java.lang.AssertionError: Expected new active leader
null
Live Nodes: [127.0.0.1:36825_solr, 127.0.0.1:41063_solr, 127.0.0.1:43487_solr]
Last available state: DocCollection(raceDeleteReplica_true//collections/raceDeleteReplica_true/state.json/11)={
  "pullReplicas":"0",
  "replicationFactor":"2",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node4":{
          "core":"raceDeleteReplica_true_shard1_replica_n1",
          "base_url":"https://127.0.0.1:44359/solr",
          "node_name":"127.0.0.1:44359_solr",
          "state":"down",
          "type":"NRT",
          "leader":"true"},
        "core_node6":{
          "core":"raceDeleteReplica_true_shard1_replica_n5",
          "base_url":"https://127.0.0.1:44359/solr",
          "node_name":"127.0.0.1:44359_solr",
          "state":"down",
          "type":"NRT"},
        "core_node3":{
          "core":"raceDeleteReplica_true_shard1_replica_n2",
          "base_url":"https://127.0.0.1:41063/solr",
          "node_name":"127.0.0.1:41063_solr",
          "state":"down",
          "type":"NRT"}}}},
  "router":{"name":"compositeId"},
  "maxShardsPerNode":"1",
  "autoAddReplicas":"false",
  "nrtReplicas":"2",
  "tlogReplicas":"0"}
	at __randomizedtesting.SeedInfo.seed([EF573F4625E2C3F9:85415E964D108933]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.SolrCloudTestCase.waitForState(SolrCloudTestCase.java:280)
	at org.apache.solr.cloud.DeleteReplicaTest.raceConditionOnDeleteAndRegisterReplica(DeleteReplicaTest.java:334)
	at org.apache.solr.cloud.DeleteReplicaTest.raceConditionOnDeleteAndRegisterReplica(DeleteReplicaTest.java:229)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1742)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:935)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:971)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:985)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:944)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:830)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.DeleteReplicaTest.deleteLiveReplicaTest

Error Message:
Error from server at https://127.0.0.1:36825/solr: Cannot create collection delLiveColl. Value of maxShardsPerNode is 1, and the number of nodes currently live or live and part of your createNodeSet is 3. This allows a maximum of 3 to be created. Value of numShards is 2, value of nrtReplicas is 2, value of tlogReplicas is 0 and value of pullReplicas is 0. This requires 4 shards to be created (higher than the allowed number)

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:36825/solr: Cannot create collection delLiveColl. Value of maxShardsPerNode is 1, and the number of nodes currently live or live and part of your createNodeSet is 3. This allows a maximum of 3 to be created. Value of numShards is 2, value of nrtReplicas is 2, value of tlogReplicas is 0 and value of pullReplicas is 0. This requires 4 shards to be created (higher than the allowed number)
	at __randomizedtesting.SeedInfo.seed([EF573F4625E2C3F9:42378B4D38DD6B8C]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1107)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:884)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:817)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
	at org.apache.solr.cloud.DeleteReplicaTest.deleteLiveReplicaTest(DeleteReplicaTest.java:75)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1742)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:935)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:971)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:985)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:944)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:830)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.autoscaling.sim.TestSimPolicyCloud.testCreateCollectionAddReplica

Error Message:
Timeout waiting for collection to become active Live Nodes: [127.0.0.1:10006_solr, 127.0.0.1:10008_solr, 127.0.0.1:10005_solr, 127.0.0.1:10009_solr, 127.0.0.1:10007_solr] Last available state: DocCollection(testCreateCollectionAddReplica//clusterstate.json/40)={   "replicationFactor":"1",   "pullReplicas":"0",   "router":{"name":"compositeId"},   "maxShardsPerNode":"1",   "autoAddReplicas":"false",   "nrtReplicas":"1",   "tlogReplicas":"0",   "autoCreated":"true",   "policy":"c1",   "shards":{"shard1":{       "replicas":{"core_node1":{           "core":"testCreateCollectionAddReplica_shard1_replica_n1",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":10240,           "node_name":"127.0.0.1:10008_solr",           "state":"active",           "type":"NRT",           "INDEX.sizeInGB":9.5367431640625E-6,           "SEARCHER.searcher.numDocs":0}},       "range":"80000000-7fffffff",       "state":"active"}}}

Stack Trace:
java.lang.AssertionError: Timeout waiting for collection to become active
Live Nodes: [127.0.0.1:10006_solr, 127.0.0.1:10008_solr, 127.0.0.1:10005_solr, 127.0.0.1:10009_solr, 127.0.0.1:10007_solr]
Last available state: DocCollection(testCreateCollectionAddReplica//clusterstate.json/40)={
  "replicationFactor":"1",
  "pullReplicas":"0",
  "router":{"name":"compositeId"},
  "maxShardsPerNode":"1",
  "autoAddReplicas":"false",
  "nrtReplicas":"1",
  "tlogReplicas":"0",
  "autoCreated":"true",
  "policy":"c1",
  "shards":{"shard1":{
      "replicas":{"core_node1":{
          "core":"testCreateCollectionAddReplica_shard1_replica_n1",
          "SEARCHER.searcher.maxDoc":0,
          "SEARCHER.searcher.deletedDocs":0,
          "INDEX.sizeInBytes":10240,
          "node_name":"127.0.0.1:10008_solr",
          "state":"active",
          "type":"NRT",
          "INDEX.sizeInGB":9.5367431640625E-6,
          "SEARCHER.searcher.numDocs":0}},
      "range":"80000000-7fffffff",
      "state":"active"}}}
	at __randomizedtesting.SeedInfo.seed([EF573F4625E2C3F9:6F775A6834A12B5F]:0)
	at org.apache.solr.cloud.CloudTestUtils.waitForState(CloudTestUtils.java:70)
	at org.apache.solr.cloud.autoscaling.sim.TestSimPolicyCloud.testCreateCollectionAddReplica(TestSimPolicyCloud.java:123)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1742)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:935)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:971)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:985)
	at org.junit.rules.ExpectedException$ExpectedExceptionStatement.evaluate(ExpectedException.java:110)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:944)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:830)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.util.concurrent.TimeoutException: last ClusterState: znodeVersion: 41
live nodes:[127.0.0.1:10006_solr, 127.0.0.1:10008_solr, 127.0.0.1:10005_solr, 127.0.0.1:10009_solr, 127.0.0.1:10007_solr]
collections:{testCreateCollectionAddReplica=DocCollection(testCreateCollectionAddReplica//clusterstate.json/40)={
  "replicationFactor":"1",
  "pullReplicas":"0",
  "router":{"name":"compositeId"},
  "maxShardsPerNode":"1",
  "autoAddReplicas":"false",
  "nrtReplicas":"1",
  "tlogReplicas":"0",
  "autoCreated":"true",
  "policy":"c1",
  "shards":{"shard1":{
      "replicas":{"core_node1":{
          "core":"testCreateCollectionAddReplica_shard1_replica_n1",
          "SEARCHER.searcher.maxDoc":0,
          "SEARCHER.searcher.deletedDocs":0,
          "INDEX.sizeInBytes":10240,
          "node_name":"127.0.0.1:10008_solr",
          "state":"active",
          "type":"NRT",
          "INDEX.sizeInGB":9.5367431640625E-6,
          "SEARCHER.searcher.numDocs":0}},
      "range":"80000000-7fffffff",
      "state":"active"}}}}, last coll state: DocCollection(testCreateCollectionAddReplica//clusterstate.json/40)={
  "replicationFactor":"1",
  "pullReplicas":"0",
  "router":{"name":"compositeId"},
  "maxShardsPerNode":"1",
  "autoAddReplicas":"false",
  "nrtReplicas":"1",
  "tlogReplicas":"0",
  "autoCreated":"true",
  "policy":"c1",
  "shards":{"shard1":{
      "replicas":{"core_node1":{
          "core":"testCreateCollectionAddReplica_shard1_replica_n1",
          "SEARCHER.searcher.maxDoc":0,
          "SEARCHER.searcher.deletedDocs":0,
          "INDEX.sizeInBytes":10240,
          "node_name":"127.0.0.1:10008_solr",
          "state":"active",
          "type":"NRT",
          "INDEX.sizeInGB":9.5367431640625E-6,
          "SEARCHER.searcher.numDocs":0}},
      "range":"80000000-7fffffff",
      "state":"active"}}}
	at org.apache.solr.cloud.CloudTestUtils.waitForState(CloudTestUtils.java:112)
	at org.apache.solr.cloud.CloudTestUtils.waitForState(CloudTestUtils.java:64)
	... 41 more


FAILED:  org.apache.solr.cloud.autoscaling.sim.TestSimPolicyCloud.testCreateCollectionSplitShard

Error Message:
java.util.concurrent.ExecutionException: org.apache.solr.common.SolrException: java.util.concurrent.TimeoutException: last ClusterState: znodeVersion: 10 live nodes:[127.0.0.1:10219_solr, 127.0.0.1:10218_solr, 127.0.0.1:10216_solr, 127.0.0.1:10217_solr, 127.0.0.1:10215_solr] collections:{testCreateCollectionSplitShard=DocCollection(testCreateCollectionSplitShard//clusterstate.json/9)={   "replicationFactor":"1",   "pullReplicas":"0",   "router":{"name":"compositeId"},   "maxShardsPerNode":"1",   "autoAddReplicas":"false",   "nrtReplicas":"2",   "tlogReplicas":"0",   "autoCreated":"true",   "policy":"c1",   "shards":{     "shard1":{       "replicas":{         "core_node1":{           "core":"testCreateCollectionSplitShard_shard1_replica_n1",           "leader":"true",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":10240,           "node_name":"127.0.0.1:10215_solr",           "state":"active",           "type":"NRT",           "INDEX.sizeInGB":9.5367431640625E-6,           "SEARCHER.searcher.numDocs":0},         "core_node2":{           "core":"testCreateCollectionSplitShard_shard1_replica_n2",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":10240,           "node_name":"127.0.0.1:10216_solr",           "state":"active",           "type":"NRT",           "INDEX.sizeInGB":9.5367431640625E-6,           "SEARCHER.searcher.numDocs":0}},       "range":"80000000-7fffffff",       "state":"active",       "__buffered_updates__":0},     "shard1_1":{       "parent":"shard1",       "stateTimestamp":"1540693239581044600",       "range":"0-7fffffff",       "state":"construction",       "replicas":{         "core_node10":{           "leader":"true",           "core":"testCreateCollectionSplitShard_shard1_1_replica_n9",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":10240,           "node_name":"127.0.0.1:10216_solr",           "base_url":"http://127.0.0.1:10216/solr",           "state":"active",           "type":"NRT",           "INDEX.sizeInGB":9.5367431640625E-6,           "SEARCHER.searcher.numDocs":0},         "core_node12":{           "core":"testCreateCollectionSplitShard_shard1_1_replica_n11",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":10240,           "node_name":"127.0.0.1:10215_solr",           "base_url":"http://127.0.0.1:10215/solr",           "state":"active",           "type":"NRT",           "INDEX.sizeInGB":9.5367431640625E-6,           "SEARCHER.searcher.numDocs":0}}},     "shard1_0":{       "parent":"shard1",       "stateTimestamp":"1540693239580926000",       "range":"80000000-ffffffff",       "state":"construction",       "replicas":{         "core_node6":{           "core":"testCreateCollectionSplitShard_shard1_0_replica_n5",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":20480,           "node_name":"127.0.0.1:10215_solr",           "base_url":"http://127.0.0.1:10215/solr",           "state":"active",           "type":"NRT",           "INDEX.sizeInGB":1.9073486328125E-5,           "SEARCHER.searcher.numDocs":0},         "core_node8":{           "core":"testCreateCollectionSplitShard_shard1_0_replica_n7",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":20480,           "node_name":"127.0.0.1:10216_solr",           "base_url":"http://127.0.0.1:10216/solr",           "state":"active",           "type":"NRT",           "INDEX.sizeInGB":1.9073486328125E-5,           "SEARCHER.searcher.numDocs":0}}}}}}, last coll state: DocCollection(testCreateCollectionSplitShard//clusterstate.json/9)={   "replicationFactor":"1",   "pullReplicas":"0",   "router":{"name":"compositeId"},   "maxShardsPerNode":"1",   "autoAddReplicas":"false",   "nrtReplicas":"2",   "tlogReplicas":"0",   "autoCreated":"true",   "policy":"c1",   "shards":{     "shard1":{       "replicas":{         "core_node1":{           "core":"testCreateCollectionSplitShard_shard1_replica_n1",           "leader":"true",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":10240,           "node_name":"127.0.0.1:10215_solr",           "state":"active",           "type":"NRT",           "INDEX.sizeInGB":9.5367431640625E-6,           "SEARCHER.searcher.numDocs":0},         "core_node2":{           "core":"testCreateCollectionSplitShard_shard1_replica_n2",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":10240,           "node_name":"127.0.0.1:10216_solr",           "state":"active",           "type":"NRT",           "INDEX.sizeInGB":9.5367431640625E-6,           "SEARCHER.searcher.numDocs":0}},       "range":"80000000-7fffffff",       "state":"active",       "__buffered_updates__":0},     "shard1_1":{       "parent":"shard1",       "stateTimestamp":"1540693239581044600",       "range":"0-7fffffff",       "state":"construction",       "replicas":{         "core_node10":{           "leader":"true",           "core":"testCreateCollectionSplitShard_shard1_1_replica_n9",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":10240,           "node_name":"127.0.0.1:10216_solr",           "base_url":"http://127.0.0.1:10216/solr",           "state":"active",           "type":"NRT",           "INDEX.sizeInGB":9.5367431640625E-6,           "SEARCHER.searcher.numDocs":0},         "core_node12":{           "core":"testCreateCollectionSplitShard_shard1_1_replica_n11",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":10240,           "node_name":"127.0.0.1:10215_solr",           "base_url":"http://127.0.0.1:10215/solr",           "state":"active",           "type":"NRT",           "INDEX.sizeInGB":9.5367431640625E-6,           "SEARCHER.searcher.numDocs":0}}},     "shard1_0":{       "parent":"shard1",       "stateTimestamp":"1540693239580926000",       "range":"80000000-ffffffff",       "state":"construction",       "replicas":{         "core_node6":{           "core":"testCreateCollectionSplitShard_shard1_0_replica_n5",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":20480,           "node_name":"127.0.0.1:10215_solr",           "base_url":"http://127.0.0.1:10215/solr",           "state":"active",           "type":"NRT",           "INDEX.sizeInGB":1.9073486328125E-5,           "SEARCHER.searcher.numDocs":0},         "core_node8":{           "core":"testCreateCollectionSplitShard_shard1_0_replica_n7",           "SEARCHER.searcher.maxDoc":0,           "SEARCHER.searcher.deletedDocs":0,           "INDEX.sizeInBytes":20480,           "node_name":"127.0.0.1:10216_solr",           "base_url":"http://127.0.0.1:10216/solr",           "state":"active",           "type":"NRT",           "INDEX.sizeInGB":1.9073486328125E-5,           "SEARCHER.searcher.numDocs":0}}}}}

Stack Trace:
java.io.IOException: java.util.concurrent.ExecutionException: org.apache.solr.common.SolrException: java.util.concurrent.TimeoutException: last ClusterState: znodeVersion: 10
live nodes:[127.0.0.1:10219_solr, 127.0.0.1:10218_solr, 127.0.0.1:10216_solr, 127.0.0.1:10217_solr, 127.0.0.1:10215_solr]
collections:{testCreateCollectionSplitShard=DocCollection(testCreateCollectionSplitShard//clusterstate.json/9)={
  "replicationFactor":"1",
  "pullReplicas":"0",
  "router":{"name":"compositeId"},
  "maxShardsPerNode":"1",
  "autoAddReplicas":"false",
  "nrtReplicas":"2",
  "tlogReplicas":"0",
  "autoCreated":"true",
  "policy":"c1",
  "shards":{
    "shard1":{
      "replicas":{
        "core_node1":{
          "core":"testCreateCollectionSplitShard_shard1_replica_n1",
          "leader":"true",
          "SEARCHER.searcher.maxDoc":0,
          "SEARCHER.searcher.deletedDocs":0,
          "INDEX.sizeInBytes":10240,
          "node_name":"127.0.0.1:10215_solr",
          "state":"active",
          "type":"NRT",
          "INDEX.sizeInGB":9.5367431640625E-6,
          "SEARCHER.searcher.numDocs":0},
        "core_node2":{
          "core":"testCreateCollectionSplitShard_shard1_replica_n2",
          "SEARCHER.searcher.maxDoc":0,
          "SEARCHER.searcher.deletedDocs":0,
          "INDEX.sizeInBytes":10240,
          "node_name":"127.0.0.1:10216_solr",
          "state":"active",
          "type":"NRT",
          "INDEX.sizeInGB":9.5367431640625E-6,
          "SEARCHER.searcher.numDocs":0}},
      "range":"80000000-7fffffff",
      "state":"active",
      "__buffered_updates__":0},
    "shard1_1":{
      "parent":"shard1",
      "stateTimestamp":"1540693239581044600",
      "range":"0-7fffffff",
      "state":"construction",
      "replicas":{
        "core_node10":{
          "leader":"true",
          "core":"testCreateCollectionSplitShard_shard1_1_replica_n9",
          "SEARCHER.searcher.maxDoc":0,
          "SEARCHER.searcher.deletedDocs":0,
          "INDEX.sizeInBytes":10240,
          "node_name":"127.0.0.1:10216_solr",
          "base_url":"http://127.0.0.1:10216/solr",
          "state":"active",
          "type":"NRT",
          "INDEX.sizeInGB":9.5367431640625E-6,
          "SEARCHER.searcher.numDocs":0},
        "core_node12":{
          "core":"testCreateCollectionSplitShard_shard1_1_replica_n11",
          "SEARCHER.searcher.maxDoc":0,
          "SEARCHER.searcher.deletedDocs":0,
          "INDEX.sizeInBytes":10240,
          "node_name":"127.0.0.1:10215_solr",
          "base_url":"http://127.0.0.1:10215/solr",
          "state":"active",
          "type":"NRT",
          "INDEX.sizeInGB":9.5367431640625E-6,
          "SEARCHER.searcher.numDocs":0}}},
    "shard1_0":{
      "parent":"shard1",
      "stateTimestamp":"1540693239580926000",
      "range":"80000000-ffffffff",
      "state":"construction",
      "replicas":{
        "core_node6":{
          "core":"testCreateCollectionSplitShard_shard1_0_replica_n5",
          "SEARCHER.searcher.maxDoc":0,
          "SEARCHER.searcher.deletedDocs":0,
          "INDEX.sizeInBytes":20480,
          "node_name":"127.0.0.1:10215_solr",
          "base_url":"http://127.0.0.1:10215/solr",
          "state":"active",
          "type":"NRT",
          "INDEX.sizeInGB":1.9073486328125E-5,
          "SEARCHER.searcher.numDocs":0},
        "core_node8":{
          "core":"testCreateCollectionSplitShard_shard1_0_replica_n7",
          "SEARCHER.searcher.maxDoc":0,
          "SEARCHER.searcher.deletedDocs":0,
          "INDEX.sizeInBytes":20480,
          "node_name":"127.0.0.1:10216_solr",
          "base_url":"http://127.0.0.1:10216/solr",
          "state":"active",
          "type":"NRT",
          "INDEX.sizeInGB":1.9073486328125E-5,
          "SEARCHER.searcher.numDocs":0}}}}}}, last coll state: DocCollection(testCreateCollectionSplitShard//clusterstate.json/9)={
  "replicationFactor":"1",
  "pullReplicas":"0",
  "router":{"name":"compositeId"},
  "maxShardsPerNode":"1",
  "autoAddReplicas":"false",
  "nrtReplicas":"2",
  "tlogReplicas":"0",
  "autoCreated":"true",
  "policy":"c1",
  "shards":{
    "shard1":{
      "replicas":{
        "core_node1":{
          "core":"testCreateCollectionSplitShard_shard1_replica_n1",
          "leader":"true",
          "SEARCHER.searcher.maxDoc":0,
          "SEARCHER.searcher.deletedDocs":0,
          "INDEX.sizeInBytes":10240,
          "node_name":"127.0.0.1:10215_solr",
          "state":"active",
          "type":"NRT",
          "INDEX.sizeInGB":9.5367431640625E-6,
          "SEARCHER.searcher.numDocs":0},
        "core_node2":{
          "core":"testCreateCollectionSplitShard_shard1_replica_n2",
          "SEARCHER.searcher.maxDoc":0,
          "SEARCHER.searcher.deletedDocs":0,
          "INDEX.sizeInBytes":10240,
          "node_name":"127.0.0.1:10216_solr",
          "state":"active",
          "type":"NRT",
          "INDEX.sizeInGB":9.5367431640625E-6,
          "SEARCHER.searcher.numDocs":0}},
      "range":"80000000-7fffffff",
      "state":"active",
      "__buffered_updates__":0},
    "shard1_1":{
      "parent":"shard1",
      "stateTimestamp":"1540693239581044600",
      "range":"0-7fffffff",
      "state":"construction",
      "replicas":{
        "core_node10":{
          "leader":"true",
          "core":"testCreateCollectionSplitShard_shard1_1_replica_n9",
          "SEARCHER.searcher.maxDoc":0,
          "SEARCHER.searcher.deletedDocs":0,
          "INDEX.sizeInBytes":10240,
          "node_name":"127.0.0.1:10216_solr",
          "base_url":"http://127.0.0.1:10216/solr",
          "state":"active",
          "type":"NRT",
          "INDEX.sizeInGB":9.5367431640625E-6,
          "SEARCHER.searcher.numDocs":0},
        "core_node12":{
          "core":"testCreateCollectionSplitShard_shard1_1_replica_n11",
          "SEARCHER.searcher.maxDoc":0,
          "SEARCHER.searcher.deletedDocs":0,
          "INDEX.sizeInBytes":10240,
          "node_name":"127.0.0.1:10215_solr",
          "base_url":"http://127.0.0.1:10215/solr",
          "state":"active",
          "type":"NRT",
          "INDEX.sizeInGB":9.5367431640625E-6,
          "SEARCHER.searcher.numDocs":0}}},
    "shard1_0":{
      "parent":"shard1",
      "stateTimestamp":"1540693239580926000",
      "range":"80000000-ffffffff",
      "state":"construction",
      "replicas":{
        "core_node6":{
          "core":"testCreateCollectionSplitShard_shard1_0_replica_n5",
          "SEARCHER.searcher.maxDoc":0,
          "SEARCHER.searcher.deletedDocs":0,
          "INDEX.sizeInBytes":20480,
          "node_name":"127.0.0.1:10215_solr",
          "base_url":"http://127.0.0.1:10215/solr",
          "state":"active",
          "type":"NRT",
          "INDEX.sizeInGB":1.9073486328125E-5,
          "SEARCHER.searcher.numDocs":0},
        "core_node8":{
          "core":"testCreateCollectionSplitShard_shard1_0_replica_n7",
          "SEARCHER.searcher.maxDoc":0,
          "SEARCHER.searcher.deletedDocs":0,
          "INDEX.sizeInBytes":20480,
          "node_name":"127.0.0.1:10216_solr",
          "base_url":"http://127.0.0.1:10216/solr",
          "state":"active",
          "type":"NRT",
          "INDEX.sizeInGB":1.9073486328125E-5,
          "SEARCHER.searcher.numDocs":0}}}}}
	at __randomizedtesting.SeedInfo.seed([EF573F4625E2C3F9:3AD94CBD95A84247]:0)
	at org.apache.solr.cloud.autoscaling.sim.SimCloudManager.request(SimCloudManager.java:654)
	at org.apache.solr.cloud.autoscaling.sim.SimCloudManager$1.request(SimCloudManager.java:224)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
	at org.apache.solr.cloud.autoscaling.sim.TestSimPolicyCloud.testCreateCollectionSplitShard(TestSimPolicyCloud.java:168)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1742)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:935)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:971)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:985)
	at org.junit.rules.ExpectedException$ExpectedExceptionStatement.evaluate(ExpectedException.java:110)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:944)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:830)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.util.concurrent.ExecutionException: org.apache.solr.common.SolrException: java.util.concurrent.TimeoutException: last ClusterState: znodeVersion: 10
live nodes:[127.0.0.1:10219_solr, 127.0.0.1:10218_solr, 127.0.0.1:10216_solr, 127.0.0.1:10217_solr, 127.0.0.1:10215_solr]
collections:{testCreateCollectionSplitShard=DocCollection(testCreateCollectionSplitShard//clusterstate.json/9)={
  "replicationFactor":"1",
  "pullReplicas":"0",
  "router":{"name":"compositeId"},
  "maxShardsPerNode":"1",
  "autoAddReplicas":"false",
  "nrtReplicas":"2",
  "tlogReplicas":"0",
  "autoCreated":"true",
  "policy":"c1",
  "shards":{
    "shard1":{
      "replicas":{
        "core_node1":{
          "core":"testCreateCollectionSplitShard_shard1_replica_n1",
          "leader":"true",
          "SEARCHER.searcher.maxDoc":0,
          "SEARCHER.searcher.deletedDocs":0,
          "INDEX.sizeInBytes":10240,
          "node_name":"127.0.0.1:10215_solr",
          "state":"active",
          "type":"NRT",
          "INDEX.sizeInGB":9.5367431640625E-6,
          "SEARCHER.searcher.numDocs":0},
        "core_node2":{
          "core":"testCreateCollectionSplitShard_shard1_replica_n2",
          "SEARCHER.searcher.maxDoc":0,
          "SEARCHER.searcher.deletedDocs":0,
          "INDEX.sizeInBytes":10240,
          "node_name":"127.0.0.1:10216_solr",
          "state":"active",
          "type":"NRT",
          "INDEX.sizeInGB":9.5367431640625E-6,
          "SEARCHER.searcher.numDocs":0}},
      "range":"80000000-7fffffff",
      "state":"active",
      "__buffered_updates__":0},
    "shard1_1":{
      "parent":"shard1",
      "stateTimestamp":"1540693239581044600",
      "range":"0-7fffffff",
      "state":"construction",
      "replicas":{
        "core_node10":{
          "leader":"true",
          "core":"testCreateCollectionSplitShard_shard1_1_replica_n9",
          "SEARCHER.searcher.maxDoc":0,
          "SEARCHER.searcher.deletedDocs":0,
          "INDEX.sizeInBytes":10240,
          "node_name":"127.0.0.1:10216_solr",
          "base_url":"http://127.0.0.1:10216/solr",
          "state":"active",
          "type":"NRT",
          "INDEX.sizeInGB":9.5367431640625E-6,
          "SEARCHER.searcher.numDocs":0},
        "core_node12":{
          "core":"testCreateCollectionSplitShard_shard1_1_replica_n11",
          "SEARCHER.searcher.maxDoc":0,
          "SEARCHER.searcher.deletedDocs":0,
          "INDEX.sizeInBytes":10240,
          "node_name":"127.0.0.1:10215_solr",
          "base_url":"http://127.0.0.1:10215/solr",
          "state":"active",
          "type":"NRT",
          "INDEX.sizeInGB":9.5367431640625E-6,
          "SEARCHER.searcher.numDocs":0}}},
    "shard1_0":{
      "parent":"shard1",
      "stateTimestamp":"1540693239580926000",
      "range":"80000000-ffffffff",
      "state":"construction",
      "replicas":{
        "core_node6":{
          "core":"testCreateCollectionSplitShard_shard1_0_replica_n5",
          "SEARCHER.searcher.maxDoc":0,
          "SEARCHER.searcher.deletedDocs":0,
          "INDEX.sizeInBytes":20480,
          "node_name":"127.0.0.1:10215_solr",
          "base_url":"http://127.0.0.1:10215/solr",
          "state":"active",
          "type":"NRT",
          "INDEX.sizeInGB":1.9073486328125E-5,
          "SEARCHER.searcher.numDocs":0},
        "core_node8":{
          "core":"testCreateCollectionSplitShard_shard1_0_replica_n7",
          "SEARCHER.searcher.maxDoc":0,
          "SEARCHER.searcher.deletedDocs":0,
          "INDEX.sizeInBytes":20480,
          "node_name":"127.0.0.1:10216_solr",
          "base_url":"http://127.0.0.1:10216/solr",
          "state":"active",
          "type":"NRT",
          "INDEX.sizeInGB":1.9073486328125E-5,
          "SEARCHER.searcher.numDocs":0}}}}}}, last coll state: DocCollection(testCreateCollectionSplitShard//clusterstate.json/9)={
  "replicationFactor":"1",
  "pullReplicas":"0",
  "router":{"name":"compositeId"},
  "maxShardsPerNode":"1",
  "autoAddReplicas":"false",
  "nrtReplicas":"2",
  "tlogReplicas":"0",
  "autoCreated":"true",
  "policy":"c1",
  "shards":{
    "shard1":{
      "replicas":{
        "core_node1":{
          "core":"testCreateCollectionSplitShard_shard1_replica_n1",
          "leader":"true",
          "SEARCHER.searcher.maxDoc":0,
          "SEARCHER.searcher.deletedDocs":0,
          "INDEX.sizeInBytes":10240,
          "node_name":"127.0.0.1:10215_solr",
          "state":"active",
          "type":"NRT",
          "INDEX.sizeInGB":9.5367431640625E-6,
          "SEARCHER.searcher.numDocs":0},
        "core_node2":{
          "core":"testCreateCollectionSplitShard_shard1_replica_n2",
          "SEARCHER.searcher.maxDoc":0,
          "SEARCHER.searcher.deletedDocs":0,
          "INDEX.sizeInBytes":10240,
          "node_name":"127.0.0.1:10216_solr",
          "state":"active",
          "type":"NRT",
          "INDEX.sizeInGB":9.5367431640625E-6,
          "SEARCHER.searcher.numDocs":0}},
      "range":"80000000-7fffffff",
      "state":"active",
      "__buffered_updates__":0},
    "shard1_1":{
      "parent":"shard1",
      "stateTimestamp":"1540693239581044600",
      "range":"0-7fffffff",
      "state":"construction",
      "replicas":{
        "core_node10":{
          "leader":"true",
          "core":"testCreateCollectionSplitShard_shard1_1_replica_n9",
          "SEARCHER.searcher.maxDoc":0,
          "SEARCHER.searcher.deletedDocs":0,
          "INDEX.sizeInBytes":10240,
          "node_name":"127.0.0.1:10216_solr",
          "base_url":"http://127.0.0.1:10216/solr",
          "state":"active",
          "type":"NRT",
          "INDEX.sizeInGB":9.5367431640625E-6,
          "SEARCHER.searcher.numDocs":0},
        "core_node12":{
          "core":"testCreateCollectionSplitShard_shard1_1_replica_n11",
          "SEARCHER.searcher.maxDoc":0,
          "SEARCHER.searcher.deletedDocs":0,
          "INDEX.sizeInBytes":10240,
          "node_name":"127.0.0.1:10215_solr",
          "base_url":"http://127.0.0.1:10215/solr",
          "state":"active",
          "type":"NRT",
          "INDEX.sizeInGB":9.5367431640625E-6,
          "SEARCHER.searcher.numDocs":0}}},
    "shard1_0":{
      "parent":"shard1",
      "stateTimestamp":"1540693239580926000",
      "range":"80000000-ffffffff",
      "state":"construction",
      "replicas":{
        "core_node6":{
          "core":"testCreateCollectionSplitShard_shard1_0_replica_n5",
          "SEARCHER.searcher.maxDoc":0,
          "SEARCHER.searcher.deletedDocs":0,
          "INDEX.sizeInBytes":20480,
          "node_name":"127.0.0.1:10215_solr",
          "base_url":"http://127.0.0.1:10215/solr",
          "state":"active",
          "type":"NRT",
          "INDEX.sizeInGB":1.9073486328125E-5,
          "SEARCHER.searcher.numDocs":0},
        "core_node8":{
          "core":"testCreateCollectionSplitShard_shard1_0_replica_n7",
          "SEARCHER.searcher.maxDoc":0,
          "SEARCHER.searcher.deletedDocs":0,
          "INDEX.sizeInBytes":20480,
          "node_name":"127.0.0.1:10216_solr",
          "base_url":"http://127.0.0.1:10216/solr",
          "state":"active",
          "type":"NRT",
          "INDEX.sizeInGB":1.9073486328125E-5,
          "SEARCHER.searcher.numDocs":0}}}}}
	at java.util.concurrent.FutureTask.report(FutureTask.java:122)
	at java.util.concurrent.FutureTask.get(FutureTask.java:192)
	at org.apache.solr.cloud.autoscaling.sim.SimCloudManager.request(SimCloudManager.java:652)
	... 44 more
Caused by: org.apache.solr.common.SolrException: java.util.concurrent.TimeoutException: last ClusterState: znodeVersion: 10
live nodes:[127.0.0.1:10219_solr, 127.0.0.1:10218_solr, 127.0.0.1:10216_solr, 127.0.0.1:10217_solr, 127.0.0.1:10215_solr]
collections:{testCreateCollectionSplitShard=DocCollection(testCreateCollectionSplitShard//clusterstate.json/9)={
  "replicationFactor":"1",
  "pullReplicas":"0",
  "router":{"name":"compositeId"},
  "maxShardsPerNode":"1",
  "autoAddReplicas":"false",
  "nrtReplicas":"2",
  "tlogReplicas":"0",
  "autoCreated":"true",
  "policy":"c1",
  "shards":{
    "shard1":{
      "replicas":{
        "core_node1":{
          "core":"testCreateCollectionSplitShard_shard1_replica_n1",
          "leader":"true",
          "SEARCHER.searcher.maxDoc":0,
          "SEARCHER.searcher.deletedDocs":0,
          "INDEX.sizeInBytes":10240,
          "node_name":"127.0.0.1:10215_solr",
          "state":"active",
          "type":"NRT",
          "INDEX.sizeInGB":9.5367431640625E-6,
          "SEARCHER.searcher.numDocs":0},
        "core_node2":{
          "core":"testCreateCollectionSplitShard_shard1_replica_n2",
          "SEARCHER.searcher.maxDoc":0,
          "SEARCHER.searcher.deletedDocs":0,
          "INDEX.sizeInBytes":10240,
          "node_name":"127.0.0.1:10216_solr",
          "state":"active",
          "type":"NRT",
          "INDEX.sizeInGB":9.5367431640625E-6,
          "SEARCHER.searcher.numDocs":0}},
      "range":"80000000-7fffffff",
      "state":"active",
      "__buffered_updates__":0},
    "shard1_1":{
      "parent":"shard1",
      "stateTimestamp":"1540693239581044600",
      "range":"0-7fffffff",
      "state":"construction",
      "replicas":{
        "core_node10":{
          "leader":"true",
          "core":"testCreateCollectionSplitShard_shard1_1_replica_n9",
          "SEARCHER.searcher.maxDoc":0,
          "SEARCHER.searcher.deletedDocs":0,
          "INDEX.sizeInBytes":10240,
          "node_name":"127.0.0.1:10216_solr",
          "base_url":"http://127.0.0.1:10216/solr",
          "state":"active",
          "type":"NRT",
          "INDEX.sizeInGB":9.5367431640625E-6,
          "SEARCHER.searcher.numDocs":0},
        "core_node12":{
          "core":"testCreateCollectionSplitShard_shard1_1_replica_n11",
          "SEARCHER.searcher.maxDoc":0,
          "SEARCHER.searcher.deletedDocs":0,
          "INDEX.sizeInBytes":10240,
          "node_name":"127.0.0.1:10215_solr",
          "base_url":"http://127.0.0.1:10215/solr",
          "state":"active",
          "type":"NRT",
          "INDEX.sizeInGB":9.5367431640625E-6,
          "SEARCHER.searcher.numDocs":0}}},
    "shard1_0":{
      "parent":"shard1",
      "stateTimestamp":"1540693239580926000",
      "range":"80000000-ffffffff",
      "state":"construction",
      "replicas":{
        "core_node6":{
          "core":"testCreateCollectionSplitShard_shard1_0_replica_n5",
          "SEARCHER.searcher.maxDoc":0,
          "SEARCHER.searcher.deletedDocs":0,
          "INDEX.sizeInBytes":20480,
          "node_name":"127.0.0.1:10215_solr",
          "base_url":"http://127.0.0.1:10215/solr",
          "state":"active",
          "type":"NRT",
          "INDEX.sizeInGB":1.9073486328125E-5,
          "SEARCHER.searcher.numDocs":0},
        "core_node8":{
          "core":"testCreateCollectionSplitShard_shard1_0_replica_n7",
          "SEARCHER.searcher.maxDoc":0,
          "SEARCHER.searcher.deletedDocs":0,
          "INDEX.sizeInBytes":20480,
          "node_name":"127.0.0.1:10216_solr",
          "base_url":"http://127.0.0.1:10216/solr",
          "state":"active",
          "type":"NRT",
          "INDEX.sizeInGB":1.9073486328125E-5,
          "SEARCHER.searcher.numDocs":0}}}}}}, last coll state: DocCollection(testCreateCollectionSplitShard//clusterstate.json/9)={
  "replicationFactor":"1",
  "pullReplicas":"0",
  "router":{"name":"compositeId"},
  "maxShardsPerNode":"1",
  "autoAddReplicas":"false",
  "nrtReplicas":"2",
  "tlogReplicas":"0",
  "autoCreated":"true",
  "policy":"c1",
  "shards":{
    "shard1":{
      "replicas":{
        "core_node1":{
          "core":"testCreateCollectionSplitShard_shard1_replica_n1",
          "leader":"true",
          "SEARCHER.searcher.maxDoc":0,
          "SEARCHER.searcher.deletedDocs":0,
          "INDEX.sizeInBytes":10240,
          "node_name":"127.0.0.1:10215_solr",
          "state":"active",
          "type":"NRT",
          "INDEX.sizeInGB":9.5367431640625E-6,
          "SEARCHER.searcher.numDocs":0},
        "core_node2":{
          "core":"testCreateCollectionSplitShard_shard1_replica_n2",
          "SEARCHER.searcher.maxDoc":0,
          "SEARCHER.searcher.deletedDocs":0,
          "INDEX.sizeInBytes":10240,
          "node_name":"127.0.0.1:10216_solr",
          "state":"active",
          "type":"NRT",
          "INDEX.sizeInGB":9.5367431640625E-6,
          "SEARCHER.searcher.numDocs":0}},
      "range":"80000000-7fffffff",
      "state":"active",
      "__buffered_updates__":0},
    "shard1_1":{
      "parent":"shard1",
      "stateTimestamp":"1540693239581044600",
      "range":"0-7fffffff",
      "state":"construction",
      "replicas":{
        "core_node10":{
          "leader":"true",
          "core":"testCreateCollectionSplitShard_shard1_1_replica_n9",
          "SEARCHER.searcher.maxDoc":0,
          "SEARCHER.searcher.deletedDocs":0,
          "INDEX.sizeInBytes":10240,
          "node_name":"127.0.0.1:10216_solr",
          "base_url":"http://127.0.0.1:10216/solr",
          "state":"active",
          "type":"NRT",
          "INDEX.sizeInGB":9.5367431640625E-6,
          "SEARCHER.searcher.numDocs":0},
        "core_node12":{
          "core":"testCreateCollectionSplitShard_shard1_1_replica_n11",
          "SEARCHER.searcher.maxDoc":0,
          "SEARCHER.searcher.deletedDocs":0,
          "INDEX.sizeInBytes":10240,
          "node_name":"127.0.0.1:10215_solr",
          "base_url":"http://127.0.0.1:10215/solr",
          "state":"active",
          "type":"NRT",
          "INDEX.sizeInGB":9.5367431640625E-6,
          "SEARCHER.searcher.numDocs":0}}},
    "shard1_0":{
      "parent":"shard1",
      "stateTimestamp":"1540693239580926000",
      "range":"80000000-ffffffff",
      "state":"construction",
      "replicas":{
        "core_node6":{
          "core":"testCreateCollectionSplitShard_shard1_0_replica_n5",
          "SEARCHER.searcher.maxDoc":0,
          "SEARCHER.searcher.deletedDocs":0,
          "INDEX.sizeInBytes":20480,
          "node_name":"127.0.0.1:10215_solr",
          "base_url":"http://127.0.0.1:10215/solr",
          "state":"active",
          "type":"NRT",
          "INDEX.sizeInGB":1.9073486328125E-5,
          "SEARCHER.searcher.numDocs":0},
        "core_node8":{
          "core":"testCreateCollectionSplitShard_shard1_0_replica_n7",
          "SEARCHER.searcher.maxDoc":0,
          "SEARCHER.searcher.deletedDocs":0,
          "INDEX.sizeInBytes":20480,
          "node_name":"127.0.0.1:10216_solr",
          "base_url":"http://127.0.0.1:10216/solr",
          "state":"active",
          "type":"NRT",
          "INDEX.sizeInGB":1.9073486328125E-5,
          "SEARCHER.searcher.numDocs":0}}}}}
	at org.apache.solr.cloud.autoscaling.sim.SimCloudManager.simHandleSolrRequest(SimCloudManager.java:854)
	at org.apache.solr.cloud.autoscaling.sim.SimCloudManager.lambda$request$15(SimCloudManager.java:651)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	... 1 more
Caused by: java.util.concurrent.TimeoutException: last ClusterState: znodeVersion: 10
live nodes:[127.0.0.1:10219_solr, 127.0.0.1:10218_solr, 127.0.0.1:10216_solr, 127.0.0.1:10217_solr, 127.0.0.1:10215_solr]
collections:{testCreateCollectionSplitShard=DocCollection(testCreateCollectionSplitShard//clusterstate.json/9)={
  "replicationFactor":"1",
  "pullReplicas":"0",
  "router":{"name":"compositeId"},
  "maxShardsPerNode":"1",
  "autoAddReplicas":"false",
  "nrtReplicas":"2",
  "tlogReplicas":"0",
  "autoCreated":"true",
  "policy":"c1",
  "shards":{
    "shard1":{
      "replicas":{
        "core_node1":{
          "core":"testCreateCollectionSplitShard_shard1_replica_n1",
          "leader":"true",
          "SEARCHER.searcher.maxDoc":0,
          "SEARCHER.searcher.deletedDocs":0,
          "INDEX.sizeInBytes":10240,
          "node_name":"127.0.0.1:10215_solr",
          "state":"active",
          "type":"NRT",
          "INDEX.sizeInGB":9.5367431640625E-6,
          "SEARCHER.searcher.numDocs":0},
        "core_node2":{
          "core":"testCreateCollectionSplitShard_shard1_replica_n2",
          "SEARCHER.searcher.maxDoc":0,
          "SEARCHER.searcher.deletedDocs":0,
          "INDEX.sizeInBytes":10240,
          "node_name":"127.0.0.1:10216_solr",
          "state":"active",
          "type":"NRT",
          "INDEX.sizeInGB":9.5367431640625E-6,
          "SEARCHER.searcher.numDocs":0}},
      "range":"80000000-7fffffff",
      "state":"active",
      "__buffered_updates__":0},
    "shard1_1":{
      "parent":"shard1",
      "stateTimestamp":"1540693239581044600",
      "range":"0-7fffffff",
      "state":"construction",
      "replicas":{
        "core_node10":{
          "leader":"true",
          "core":"testCreateCollectionSplitShard_shard1_1_replica_n9",
          "SEARCHER.searcher.maxDoc":0,
          "SEARCHER.searcher.deletedDocs":0,
          "INDEX.sizeInBytes":10240,
          "node_name":"127.0.0.1:10216_solr",
          "base_url":"http://127.0.0.1:10216/solr",
          "state":"active",
          "type":"NRT",
          "INDEX.sizeInGB":9.5367431640625E-6,
          "SEARCHER.searcher.numDocs":0},
        "core_node12":{
          "core":"testCreateCollectionSplitShard_shard1_1_replica_n11",
          "SEARCHER.searcher.maxDoc":0,
          "SEARCHER.searcher.deletedDocs":0,
          "INDEX.sizeInBytes":10240,
          "node_name":"127.0.0.1:10215_solr",
          "base_url":"http://127.0.0.1:10215/solr",
          "state":"active",
          "type":"NRT",
          "INDEX.sizeInGB":9.5367431640625E-6,
          "SEARCHER.searcher.numDocs":0}}},
    "shard1_0":{
      "parent":"shard1",
      "stateTimestamp":"1540693239580926000",
      "range":"80000000-ffffffff",
      "state":"construction",
      "replicas":{
        "core_node6":{
          "core":"testCreateCollectionSplitShard_shard1_0_replica_n5",
          "SEARCHER.searcher.maxDoc":0,
          "SEARCHER.searcher.deletedDocs":0,
          "INDEX.sizeInBytes":20480,
          "node_name":"127.0.0.1:10215_solr",
          "base_url":"http://127.0.0.1:10215/solr",
          "state":"active",
          "type":"NRT",
          "INDEX.sizeInGB":1.9073486328125E-5,
          "SEARCHER.searcher.numDocs":0},
        "core_node8":{
          "core":"testCreateCollectionSplitShard_shard1_0_replica_n7",
          "SEARCHER.searcher.maxDoc":0,
          "SEARCHER.searcher.deletedDocs":0,
          "INDEX.sizeInBytes":20480,
          "node_name":"127.0.0.1:10216_solr",
          "base_url":"http://127.0.0.1:10216/solr",
          "state":"active",
          "type":"NRT",
          "INDEX.sizeInGB":1.9073486328125E-5,
          "SEARCHER.searcher.numDocs":0}}}}}}, last coll state: DocCollection(testCreateCollectionSplitShard//clusterstate.json/9)={
  "replicationFactor":"1",
  "pullReplicas":"0",
  "router":{"name":"compositeId"},
  "maxShardsPerNode":"1",
  "autoAddReplicas":"false",
  "nrtReplicas":"2",
  "tlogReplicas":"0",
  "autoCreated":"true",
  "policy":"c1",
  "shards":{
    "shard1":{
      "replicas":{
        "core_node1":{
          "core":"testCreateCollectionSplitShard_shard1_replica_n1",
          "leader":"true",
          "SEARCHER.searcher.maxDoc":0,
          "SEARCHER.searcher.deletedDocs":0,
          "INDEX.sizeInBytes":10240,
          "node_name":"127.0.0.1:10215_solr",
          "state":"active",
          "type":"NRT",
          "INDEX.sizeInGB":9.5367431640625E-6,
          "SEARCHER.searcher.numDocs":0},
        "core_node2":{
          "core":"testCreateCollectionSplitShard_shard1_replica_n2",
          "SEARCHER.searcher.maxDoc":0,
          "SEARCHER.searcher.deletedDocs":0,
          "INDEX.sizeInBytes":10240,
          "node_name":"127.0.0.1:10216_solr",
          "state":"active",
          "type":"NRT",
          "INDEX.sizeInGB":9.5367431640625E-6,
          "SEARCHER.searcher.numDocs":0}},
      "range":"80000000-7fffffff",
      "state":"active",
      "__buffered_updates__":0},
    "shard1_1":{
      "parent":"shard1",
      "stateTimestamp":"1540693239581044600",
      "range":"0-7fffffff",
      "state":"construction",
      "replicas":{
        "core_node10":{
          "leader":"true",
          "core":"testCreateCollectionSplitShard_shard1_1_replica_n9",
          "SEARCHER.searcher.maxDoc":0,
          "SEARCHER.searcher.deletedDocs":0,
          "INDEX.sizeInBytes":10240,
          "node_name":"127.0.0.1:10216_solr",
          "base_url":"http://127.0.0.1:10216/solr",
          "state":"active",
          "type":"NRT",
          "INDEX.sizeInGB":9.5367431640625E-6,
          "SEARCHER.searcher.numDocs":0},
        "core_node12":{
          "core":"testCreateCollectionSplitShard_shard1_1_replica_n11",
          "SEARCHER.searcher.maxDoc":0,
          "SEARCHER.searcher.deletedDocs":0,
          "INDEX.sizeInBytes":10240,
          "node_name":"127.0.0.1:10215_solr",
          "base_url":"http://127.0.0.1:10215/solr",
          "state":"active",
          "type":"NRT",
          "INDEX.sizeInGB":9.5367431640625E-6,
          "SEARCHER.searcher.numDocs":0}}},
    "shard1_0":{
      "parent":"shard1",
      "stateTimestamp":"1540693239580926000",
      "range":"80000000-ffffffff",
      "state":"construction",
      "replicas":{
        "core_node6":{
          "core":"testCreateCollectionSplitShard_shard1_0_replica_n5",
          "SEARCHER.searcher.maxDoc":0,
          "SEARCHER.searcher.deletedDocs":0,
          "INDEX.sizeInBytes":20480,
          "node_name":"127.0.0.1:10215_solr",
          "base_url":"http://127.0.0.1:10215/solr",
          "state":"active",
          "type":"NRT",
          "INDEX.sizeInGB":1.9073486328125E-5,
          "SEARCHER.searcher.numDocs":0},
        "core_node8":{
          "core":"testCreateCollectionSplitShard_shard1_0_replica_n7",
          "SEARCHER.searcher.maxDoc":0,
          "SEARCHER.searcher.deletedDocs":0,
          "INDEX.sizeInBytes":20480,
          "node_name":"127.0.0.1:10216_solr",
          "base_url":"http://127.0.0.1:10216/solr",
          "state":"active",
          "type":"NRT",
          "INDEX.sizeInGB":1.9073486328125E-5,
          "SEARCHER.searcher.numDocs":0}}}}}
	at org.apache.solr.cloud.CloudTestUtils.waitForState(CloudTestUtils.java:112)
	at org.apache.solr.cloud.autoscaling.sim.SimClusterStateProvider.simSplitShard(SimClusterStateProvider.java:1246)
	at org.apache.solr.cloud.autoscaling.sim.SimCloudManager.simHandleSolrRequest(SimCloudManager.java:852)
	... 6 more




Build Log:
[...truncated 12798 lines...]
   [junit4] Suite: org.apache.solr.cloud.DeleteReplicaTest
   [junit4]   2> 82530 INFO  (SUITE-DeleteReplicaTest-seed#[EF573F4625E2C3F9]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.DeleteReplicaTest_EF573F4625E2C3F9-001/init-core-data-001
   [junit4]   2> 82530 WARN  (SUITE-DeleteReplicaTest-seed#[EF573F4625E2C3F9]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=1 numCloses=1
   [junit4]   2> 82530 INFO  (SUITE-DeleteReplicaTest-seed#[EF573F4625E2C3F9]-worker) [    ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 82532 INFO  (SUITE-DeleteReplicaTest-seed#[EF573F4625E2C3F9]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 82533 INFO  (SUITE-DeleteReplicaTest-seed#[EF573F4625E2C3F9]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 4 servers in /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.DeleteReplicaTest_EF573F4625E2C3F9-001/tempDir-001
   [junit4]   2> 82533 INFO  (SUITE-DeleteReplicaTest-seed#[EF573F4625E2C3F9]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 82533 INFO  (Thread-150) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 82534 INFO  (Thread-150) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 82535 ERROR (Thread-150) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 82634 INFO  (SUITE-DeleteReplicaTest-seed#[EF573F4625E2C3F9]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:35917
   [junit4]   2> 82639 INFO  (zkConnectionManagerCallback-266-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 82645 INFO  (jetty-launcher-263-thread-1) [    ] o.e.j.s.Server jetty-9.4.11.v20180605; built: 2018-06-05T18:24:03.829Z; git: d5fc0523cfa96bfebfbda19606cad384d772f04c; jvm 1.8.0_172-b11
   [junit4]   2> 82646 INFO  (jetty-launcher-263-thread-2) [    ] o.e.j.s.Server jetty-9.4.11.v20180605; built: 2018-06-05T18:24:03.829Z; git: d5fc0523cfa96bfebfbda19606cad384d772f04c; jvm 1.8.0_172-b11
   [junit4]   2> 82648 INFO  (jetty-launcher-263-thread-3) [    ] o.e.j.s.Server jetty-9.4.11.v20180605; built: 2018-06-05T18:24:03.829Z; git: d5fc0523cfa96bfebfbda19606cad384d772f04c; jvm 1.8.0_172-b11
   [junit4]   2> 82667 INFO  (jetty-launcher-263-thread-2) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 82667 INFO  (jetty-launcher-263-thread-2) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 82667 INFO  (jetty-launcher-263-thread-2) [    ] o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 82672 INFO  (jetty-launcher-263-thread-4) [    ] o.e.j.s.Server jetty-9.4.11.v20180605; built: 2018-06-05T18:24:03.829Z; git: d5fc0523cfa96bfebfbda19606cad384d772f04c; jvm 1.8.0_172-b11
   [junit4]   2> 82675 INFO  (jetty-launcher-263-thread-1) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 82675 INFO  (jetty-launcher-263-thread-1) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 82675 INFO  (jetty-launcher-263-thread-1) [    ] o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 82676 INFO  (jetty-launcher-263-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7b2440{/solr,null,AVAILABLE}
   [junit4]   2> 82677 INFO  (jetty-launcher-263-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@124e237{SSL,[ssl, http/1.1]}{127.0.0.1:44359}
   [junit4]   2> 82677 INFO  (jetty-launcher-263-thread-1) [    ] o.e.j.s.Server Started @82718ms
   [junit4]   2> 82677 INFO  (jetty-launcher-263-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=44359}
   [junit4]   2> 82677 ERROR (jetty-launcher-263-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 82678 INFO  (jetty-launcher-263-thread-1) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 82678 INFO  (jetty-launcher-263-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.6.0
   [junit4]   2> 82678 INFO  (jetty-launcher-263-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 82678 INFO  (jetty-launcher-263-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 82678 INFO  (jetty-launcher-263-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-10-27T13:09:50.683Z
   [junit4]   2> 82683 INFO  (zkConnectionManagerCallback-268-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 82684 INFO  (jetty-launcher-263-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 82688 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 0x1000131d4870001, likely client has closed socket
   [junit4]   2> 82691 INFO  (jetty-launcher-263-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1a11775{/solr,null,AVAILABLE}
   [junit4]   2> 82692 INFO  (jetty-launcher-263-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@17c8046{SSL,[ssl, http/1.1]}{127.0.0.1:43487}
   [junit4]   2> 82692 INFO  (jetty-launcher-263-thread-2) [    ] o.e.j.s.Server Started @82733ms
   [junit4]   2> 82692 INFO  (jetty-launcher-263-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=43487}
   [junit4]   2> 82692 ERROR (jetty-launcher-263-thread-2) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir.

[...truncated too long message...]

r Closing metric reporters for registry=solr.core.raceDeleteReplica_true.shard1.replica_n2, tag=6af72b
   [junit4]   2> 226225 INFO  (coreCloseExecutor-422-thread-1) [n:127.0.0.1:34371_solr c:raceDeleteReplica_true s:shard1 r:core_node4 x:raceDeleteReplica_true_shard1_replica_n2] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@297c92: rootName = solr_34371, domain = solr.core.raceDeleteReplica_true.shard1.replica_n2, service url = null, agent id = null] for registry solr.core.raceDeleteReplica_true.shard1.replica_n2 / com.codahale.metrics.MetricRegistry@1ae13db
   [junit4]   2> 226232 INFO  (coreCloseExecutor-420-thread-2) [n:127.0.0.1:34053_solr c:deleteByCount s:shard1 r:core_node5 x:deleteByCount_shard1_replica_t1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.deleteByCount.shard1.leader, tag=b9e808
   [junit4]   2> 226238 INFO  (coreCloseExecutor-420-thread-1) [n:127.0.0.1:34053_solr c:deleteByCountNew s:shard2 r:core_node7 x:deleteByCountNew_shard2_replica_n4] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.deleteByCountNew.shard2.leader, tag=1f32a99
   [junit4]   2> 226242 INFO  (coreCloseExecutor-421-thread-1) [n:127.0.0.1:38881_solr c:deleteByCountNew s:shard1 r:core_node5 x:deleteByCountNew_shard1_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.deleteByCountNew.shard1.leader, tag=6fdd1d
   [junit4]   2> 226242 INFO  (coreCloseExecutor-421-thread-2) [n:127.0.0.1:38881_solr c:deletereplica_test s:shard1 r:core_node4 x:deletereplica_test_shard1_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.deletereplica_test.shard1.leader, tag=18dd104
   [junit4]   2> 226251 INFO  (coreCloseExecutor-422-thread-1) [n:127.0.0.1:34371_solr c:raceDeleteReplica_true s:shard1 r:core_node4 x:raceDeleteReplica_true_shard1_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.raceDeleteReplica_true.shard1.leader, tag=6af72b
   [junit4]   2> 226251 INFO  (coreCloseExecutor-422-thread-2) [n:127.0.0.1:34371_solr c:deleteReplicaOnIndexing s:shard1 r:core_node3 x:deleteReplicaOnIndexing_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.deleteReplicaOnIndexing.shard1.leader, tag=c8de45
   [junit4]   2> 226251 INFO  (coreCloseExecutor-422-thread-2) [n:127.0.0.1:34371_solr c:deleteReplicaOnIndexing s:shard1 r:core_node3 x:deleteReplicaOnIndexing_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 Committing on IndexWriter close.
   [junit4]   2> 226251 INFO  (coreCloseExecutor-422-thread-2) [n:127.0.0.1:34371_solr c:deleteReplicaOnIndexing s:shard1 r:core_node3 x:deleteReplicaOnIndexing_shard1_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@1ddc797 commitCommandVersion:0
   [junit4]   2> 226260 INFO  (jetty-closer-145-thread-1) [    ] o.a.s.c.Overseer Overseer (id=72059163647148038-127.0.0.1:34053_solr-n_0000000000) closing
   [junit4]   2> 226260 INFO  (OverseerStateUpdate-72059163647148038-127.0.0.1:34053_solr-n_0000000000) [n:127.0.0.1:34053_solr    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:34053_solr
   [junit4]   2> 226261 WARN  (OverseerAutoScalingTriggerThread-72059163647148038-127.0.0.1:34053_solr-n_0000000000) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 226263 INFO  (zkCallback-225-thread-2) [    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:34371_solr
   [junit4]   2> 226264 INFO  (jetty-closer-145-thread-1) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@f833e7{/solr,null,UNAVAILABLE}
   [junit4]   2> 226264 INFO  (jetty-closer-145-thread-1) [    ] o.e.j.s.session node0 Stopped scavenging
   [junit4]   2> 226285 INFO  (jetty-closer-145-thread-4) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@6e4767{/solr,null,UNAVAILABLE}
   [junit4]   2> 226286 INFO  (jetty-closer-145-thread-4) [    ] o.e.j.s.session node0 Stopped scavenging
   [junit4]   2> 227763 INFO  (jetty-closer-145-thread-2) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@4046e2{/solr,null,UNAVAILABLE}
   [junit4]   2> 227764 INFO  (jetty-closer-145-thread-2) [    ] o.e.j.s.session node0 Stopped scavenging
   [junit4]   2> 227764 ERROR (SUITE-DeleteReplicaTest-seed#[EF573F4625E2C3F9]-worker) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 227764 INFO  (SUITE-DeleteReplicaTest-seed#[EF573F4625E2C3F9]-worker) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:36043 36043
   [junit4]   2> 227810 INFO  (Thread-289) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:36043 36043
   [junit4]   2> 227811 WARN  (Thread-289) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	14	/solr/collections/deleteFromClusterState_true/terms/shard1
   [junit4]   2> 	14	/solr/collections/deleteFromClusterState_false/terms/shard1
   [junit4]   2> 	11	/solr/aliases.json
   [junit4]   2> 	11	/solr/configs/conf
   [junit4]   2> 	8	/solr/collections/raceDeleteReplica_true/terms/shard1
   [junit4]   2> 	7	/solr/collections/deleteByCountNew/terms/shard1
   [junit4]   2> 	7	/solr/collections/deleteByCountNew/terms/shard2
   [junit4]   2> 	7	/solr/collections/deletereplica_test/terms/shard1
   [junit4]   2> 	6	/solr/collections/deleteByCount/terms/shard1
   [junit4]   2> 	5	/solr/collections/deleteReplicaOnIndexing/terms/shard1
   [junit4]   2> 	5	/solr/security.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	11	/solr/clusterprops.json
   [junit4]   2> 	11	/solr/clusterstate.json
   [junit4]   2> 	8	/solr/collections/raceDeleteReplica_true/state.json
   [junit4]   2> 	5	/solr/collections/deleteFromClusterState_false/state.json
   [junit4]   2> 	5	/solr/collections/deleteByCountNew/state.json
   [junit4]   2> 	4	/solr/collections/deleteByCount/state.json
   [junit4]   2> 	4	/solr/collections/deleteFromClusterState_true/state.json
   [junit4]   2> 	3	/solr/collections/deleteReplicaOnIndexing/state.json
   [junit4]   2> 	2	/solr/overseer_elect/election/72059163647148041-127.0.0.1:38881_solr-n_0000000001
   [junit4]   2> 	2	/solr/collections/deletereplica_test/state.json
   [junit4]   2> 	2	/solr/collections/raceDeleteReplica_true/leader_elect/shard1/election/72059163647148044-core_node3-n_0000000000
   [junit4]   2> 	2	/solr/overseer_elect/election/72059163647148044-127.0.0.1:40209_solr-n_0000000002
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	11	/solr/live_nodes
   [junit4]   2> 	11	/solr/collections
   [junit4]   2> 
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.DeleteReplicaTest_EF573F4625E2C3F9-001
   [junit4]   2> Oct 27, 2018 2:17:13 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70), sim=RandomSimilarity(queryNorm=true): {}, locale=es-GT, timezone=Pacific/Chuuk
   [junit4]   2> NOTE: Linux 4.15.0-38-generic i386/Oracle Corporation 1.8.0_172 (32-bit)/cpus=8,threads=1,free=57856320,total=267649024
   [junit4]   2> NOTE: All tests run in this JVM: [DeleteReplicaTest, DeleteReplicaTest]
   [junit4] Completed [10/10 (3!)] on J1 in 136.01s, 7 tests, 1 failure, 1 error <<< FAILURES!

[...truncated 15 lines...]
BUILD FAILED
/home/jenkins/workspace/Lucene-Solr-7.x-Linux/lucene/common-build.xml:1567: The following error occurred while executing this line:
/home/jenkins/workspace/Lucene-Solr-7.x-Linux/lucene/common-build.xml:1092: There were test failures: 10 suites, 70 tests, 2 errors, 3 failures [seed: EF573F4625E2C3F9]

Total time: 3 minutes 50 seconds

[repro] Setting last failure code to 256

[repro] Failures:
[repro]   0/5 failed: org.apache.solr.client.solrj.impl.CloudSolrClientTest
[repro]   1/5 failed: org.apache.solr.cloud.autoscaling.sim.TestSimPolicyCloud
[repro]   2/5 failed: org.apache.solr.cloud.DeleteReplicaTest
[repro] Exiting with code 256
+ mv lucene/build lucene/build.repro
+ mv solr/build solr/build.repro
+ mv lucene/build.orig lucene/build
+ mv solr/build.orig solr/build
Archiving artifacts
Setting ANT_1_8_2_HOME=/var/lib/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
[WARNINGS] Parsing warnings in console log with parser Java Compiler (javac)
Setting ANT_1_8_2_HOME=/var/lib/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
Setting ANT_1_8_2_HOME=/var/lib/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
<Git Blamer> Using GitBlamer to create author and commit information for all warnings.
<Git Blamer> GIT_COMMIT=575d3dac0ff12bc66e4cd7a211661307d43f3348, workspace=/var/lib/jenkins/workspace/Lucene-Solr-7.x-Linux
[WARNINGS] Computing warning deltas based on reference build #2986
Recording test results
Setting ANT_1_8_2_HOME=/var/lib/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)
Setting ANT_1_8_2_HOME=/var/lib/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
Setting ANT_1_8_2_HOME=/var/lib/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
Setting ANT_1_8_2_HOME=/var/lib/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2
Setting ANT_1_8_2_HOME=/var/lib/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2