You are viewing a plain text version of this content. The canonical link for it is here.
Posted to builds@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2020/10/17 17:30:54 UTC

[JENKINS-EA] Lucene-Solr-master-Linux (64bit/jdk-16-ea+18) - Build # 28358 - Unstable!

Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Linux/28358/
Java: 64bit/jdk-16-ea+18 -XX:-UseCompressedOops -XX:+UseParallelGC

50 tests failed.
FAILED:  org.apache.solr.cloud.AssignBackwardCompatibilityTest.test

Error Message:
java.lang.RuntimeException: Failed while waiting for active collection
Timeout waiting to see state for collection=collection1 :DocCollection(collection1/21)={
  "pullReplicas":"0",
  "replicationFactor":"4",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node10":{
          "core":"collection1_shard1_replica_n9",
          "leader":"true",
          "core_node_name":"core_node10",
          "base_url":"http://127.0.0.1:41589/solr",
          "node_name":"127.0.0.1:41589_solr",
          "state":"active",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node12":{
          "core":"collection1_shard1_replica_n11",
          "core_node_name":"core_node12",
          "base_url":"http://127.0.0.1:37209/solr",
          "node_name":"127.0.0.1:37209_solr",
          "state":"recovering",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"4",
  "tlogReplicas":"0"}
Live Nodes: [127.0.0.1:36461_solr, 127.0.0.1:37209_solr, 127.0.0.1:41589_solr, 127.0.0.1:44947_solr]
Last available state: DocCollection(collection1/21)={
  "pullReplicas":"0",
  "replicationFactor":"4",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node10":{
          "core":"collection1_shard1_replica_n9",
          "leader":"true",
          "core_node_name":"core_node10",
          "base_url":"http://127.0.0.1:41589/solr",
          "node_name":"127.0.0.1:41589_solr",
          "state":"active",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node12":{
          "core":"collection1_shard1_replica_n11",
          "core_node_name":"core_node12",
          "base_url":"http://127.0.0.1:37209/solr",
          "node_name":"127.0.0.1:37209_solr",
          "state":"recovering",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"4",
  "tlogReplicas":"0"}

Stack Trace:
java.lang.RuntimeException: Failed while waiting for active collection
Timeout waiting to see state for collection=collection1 :DocCollection(collection1/21)={
  "pullReplicas":"0",
  "replicationFactor":"4",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node10":{
          "core":"collection1_shard1_replica_n9",
          "leader":"true",
          "core_node_name":"core_node10",
          "base_url":"http://127.0.0.1:41589/solr",
          "node_name":"127.0.0.1:41589_solr",
          "state":"active",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node12":{
          "core":"collection1_shard1_replica_n11",
          "core_node_name":"core_node12",
          "base_url":"http://127.0.0.1:37209/solr",
          "node_name":"127.0.0.1:37209_solr",
          "state":"recovering",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"4",
  "tlogReplicas":"0"}
Live Nodes: [127.0.0.1:36461_solr, 127.0.0.1:37209_solr, 127.0.0.1:41589_solr, 127.0.0.1:44947_solr]
Last available state: DocCollection(collection1/21)={
  "pullReplicas":"0",
  "replicationFactor":"4",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node10":{
          "core":"collection1_shard1_replica_n9",
          "leader":"true",
          "core_node_name":"core_node10",
          "base_url":"http://127.0.0.1:41589/solr",
          "node_name":"127.0.0.1:41589_solr",
          "state":"active",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node12":{
          "core":"collection1_shard1_replica_n11",
          "core_node_name":"core_node12",
          "base_url":"http://127.0.0.1:37209/solr",
          "node_name":"127.0.0.1:37209_solr",
          "state":"recovering",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"4",
  "tlogReplicas":"0"}
	at __randomizedtesting.SeedInfo.seed([BD7F9393C24DB6C9:352BAC496CB1DB31]:0)
	at org.apache.solr.cloud.MiniSolrCloudCluster.waitForActiveCollection(MiniSolrCloudCluster.java:774)
	at org.apache.solr.cloud.MiniSolrCloudCluster.waitForActiveCollection(MiniSolrCloudCluster.java:780)
	at org.apache.solr.cloud.AssignBackwardCompatibilityTest.test(AssignBackwardCompatibilityTest.java:95)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:64)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:564)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:832)


FAILED:  org.apache.solr.cloud.BasicDistributedZk2Test.test

Error Message:
java.lang.AssertionError: There are still nodes recovering - waited for 330SECONDS

Stack Trace:
java.lang.AssertionError: There are still nodes recovering - waited for 330SECONDS
	at __randomizedtesting.SeedInfo.seed([BD7F9393C24DB6C9:352BAC496CB1DB31]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:197)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:144)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:139)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:1051)
	at org.apache.solr.cloud.BasicDistributedZk2Test.brindDownShardIndexSomeDocsAndRecover(BasicDistributedZk2Test.java:362)
	at org.apache.solr.cloud.BasicDistributedZk2Test.test(BasicDistributedZk2Test.java:103)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:64)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:564)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1090)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1061)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:832)


FAILED:  org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest.test

Error Message:
java.util.concurrent.TimeoutException: Cluster state still in flux after 30 retry intervals.

Stack Trace:
java.util.concurrent.TimeoutException: Cluster state still in flux after 30 retry intervals.
	at __randomizedtesting.SeedInfo.seed([BD7F9393C24DB6C9:352BAC496CB1DB31]:0)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForThingsToLevelOut(AbstractFullDistribZkTestBase.java:1643)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForThingsToLevelOut(AbstractFullDistribZkTestBase.java:1613)
	at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest.test(ChaosMonkeyNothingIsSafeWithPullReplicasTest.java:253)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:64)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:564)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1090)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1061)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:832)


FAILED:  org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest.test

Error Message:
java.lang.AssertionError: There are still nodes recovering - waited for 180SECONDS

Stack Trace:
java.lang.AssertionError: There are still nodes recovering - waited for 180SECONDS
	at __randomizedtesting.SeedInfo.seed([BD7F9393C24DB6C9:352BAC496CB1DB31]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:197)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:150)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:1063)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForThingsToLevelOut(AbstractFullDistribZkTestBase.java:1621)
	at org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest.test(ChaosMonkeySafeLeaderWithPullReplicasTest.java:198)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:64)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:564)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1090)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1061)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:832)


FAILED:  org.apache.solr.cloud.DeleteReplicaTest.raceConditionOnDeleteAndRegisterReplica

Error Message:
java.lang.RuntimeException: Failed while waiting for active collection
Timeout waiting to see state for collection=raceDeleteReplicaCollection :DocCollection(raceDeleteReplicaCollection/17)={
  "pullReplicas":"0",
  "replicationFactor":"2",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node4":{
          "core":"raceDeleteReplicaCollection_shard1_replica_n2",
          "core_node_name":"core_node4",
          "base_url":"https://127.0.0.1:42415/solr",
          "node_name":"127.0.0.1:42415_solr",
          "state":"recovering",
          "collection":"raceDeleteReplicaCollection",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node6":{
          "core":"raceDeleteReplicaCollection_shard1_replica_n5",
          "leader":"true",
          "core_node_name":"core_node6",
          "base_url":"https://127.0.0.1:34061/solr",
          "node_name":"127.0.0.1:34061_solr",
          "state":"active",
          "collection":"raceDeleteReplicaCollection",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"2",
  "tlogReplicas":"0"}
Live Nodes: [127.0.0.1:34061_solr, 127.0.0.1:42415_solr, 127.0.0.1:43235_solr, 127.0.0.1:46313_solr]
Last available state: DocCollection(raceDeleteReplicaCollection/17)={
  "pullReplicas":"0",
  "replicationFactor":"2",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node4":{
          "core":"raceDeleteReplicaCollection_shard1_replica_n2",
          "core_node_name":"core_node4",
          "base_url":"https://127.0.0.1:42415/solr",
          "node_name":"127.0.0.1:42415_solr",
          "state":"recovering",
          "collection":"raceDeleteReplicaCollection",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node6":{
          "core":"raceDeleteReplicaCollection_shard1_replica_n5",
          "leader":"true",
          "core_node_name":"core_node6",
          "base_url":"https://127.0.0.1:34061/solr",
          "node_name":"127.0.0.1:34061_solr",
          "state":"active",
          "collection":"raceDeleteReplicaCollection",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"2",
  "tlogReplicas":"0"}

Stack Trace:
java.lang.RuntimeException: Failed while waiting for active collection
Timeout waiting to see state for collection=raceDeleteReplicaCollection :DocCollection(raceDeleteReplicaCollection/17)={
  "pullReplicas":"0",
  "replicationFactor":"2",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node4":{
          "core":"raceDeleteReplicaCollection_shard1_replica_n2",
          "core_node_name":"core_node4",
          "base_url":"https://127.0.0.1:42415/solr",
          "node_name":"127.0.0.1:42415_solr",
          "state":"recovering",
          "collection":"raceDeleteReplicaCollection",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node6":{
          "core":"raceDeleteReplicaCollection_shard1_replica_n5",
          "leader":"true",
          "core_node_name":"core_node6",
          "base_url":"https://127.0.0.1:34061/solr",
          "node_name":"127.0.0.1:34061_solr",
          "state":"active",
          "collection":"raceDeleteReplicaCollection",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"2",
  "tlogReplicas":"0"}
Live Nodes: [127.0.0.1:34061_solr, 127.0.0.1:42415_solr, 127.0.0.1:43235_solr, 127.0.0.1:46313_solr]
Last available state: DocCollection(raceDeleteReplicaCollection/17)={
  "pullReplicas":"0",
  "replicationFactor":"2",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node4":{
          "core":"raceDeleteReplicaCollection_shard1_replica_n2",
          "core_node_name":"core_node4",
          "base_url":"https://127.0.0.1:42415/solr",
          "node_name":"127.0.0.1:42415_solr",
          "state":"recovering",
          "collection":"raceDeleteReplicaCollection",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node6":{
          "core":"raceDeleteReplicaCollection_shard1_replica_n5",
          "leader":"true",
          "core_node_name":"core_node6",
          "base_url":"https://127.0.0.1:34061/solr",
          "node_name":"127.0.0.1:34061_solr",
          "state":"active",
          "collection":"raceDeleteReplicaCollection",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"2",
  "tlogReplicas":"0"}
	at __randomizedtesting.SeedInfo.seed([BD7F9393C24DB6C9:D769F243AABFFC03]:0)
	at org.apache.solr.cloud.MiniSolrCloudCluster.waitForActiveCollection(MiniSolrCloudCluster.java:774)
	at org.apache.solr.cloud.MiniSolrCloudCluster.waitForActiveCollection(MiniSolrCloudCluster.java:780)
	at org.apache.solr.cloud.DeleteReplicaTest.raceConditionOnDeleteAndRegisterReplica(DeleteReplicaTest.java:385)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:64)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:564)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:832)


FAILED:  org.apache.solr.cloud.HttpPartitionTest.test

Error Message:
java.lang.AssertionError: Didn't see all replicas for shard shard1 in collDoRecoveryOnRestart come up within 130000 ms! ClusterState: {
  "control_collection":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{"core_node2":{
            "core":"control_collection_shard1_replica_n1",
            "leader":"true",
            "core_node_name":"core_node2",
            "base_url":"http://127.0.0.1:42489",
            "node_name":"127.0.0.1:42489_",
            "state":"active",
            "collection":"control_collection",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"},
  "collDoRecoveryOnRestart":{
    "pullReplicas":"0",
    "replicationFactor":"2",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"collDoRecoveryOnRestart_shard1_replica_n1",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:35541",
            "node_name":"127.0.0.1:35541_",
            "state":"recovering",
            "collection":"collDoRecoveryOnRestart",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node4":{
            "core":"collDoRecoveryOnRestart_shard1_replica_n2",
            "leader":"true",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:33565",
            "node_name":"127.0.0.1:33565_",
            "state":"active",
            "collection":"collDoRecoveryOnRestart",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"2",
    "tlogReplicas":"0"},
  "collection1":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{
      "shard1":{
        "range":"80000000-ffffffff",
        "state":"active",
        "replicas":{"core_node3":{
            "core":"collection1_shard1_replica_n1",
            "leader":"true",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:39155",
            "node_name":"127.0.0.1:39155_",
            "state":"active",
            "collection":"collection1",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}},
      "shard2":{
        "range":"0-7fffffff",
        "state":"active",
        "replicas":{
          "core_node4":{
            "core":"collection1_shard2_replica_n2",
            "leader":"true",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:33565",
            "node_name":"127.0.0.1:33565_",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node6":{
            "core":"collection1_shard2_replica_n5",
            "core_node_name":"core_node6",
            "base_url":"http://127.0.0.1:35541",
            "node_name":"127.0.0.1:35541_",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"}}

Stack Trace:
java.lang.AssertionError: Didn't see all replicas for shard shard1 in collDoRecoveryOnRestart come up within 130000 ms! ClusterState: {
  "control_collection":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{"core_node2":{
            "core":"control_collection_shard1_replica_n1",
            "leader":"true",
            "core_node_name":"core_node2",
            "base_url":"http://127.0.0.1:42489",
            "node_name":"127.0.0.1:42489_",
            "state":"active",
            "collection":"control_collection",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"},
  "collDoRecoveryOnRestart":{
    "pullReplicas":"0",
    "replicationFactor":"2",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"collDoRecoveryOnRestart_shard1_replica_n1",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:35541",
            "node_name":"127.0.0.1:35541_",
            "state":"recovering",
            "collection":"collDoRecoveryOnRestart",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node4":{
            "core":"collDoRecoveryOnRestart_shard1_replica_n2",
            "leader":"true",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:33565",
            "node_name":"127.0.0.1:33565_",
            "state":"active",
            "collection":"collDoRecoveryOnRestart",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"2",
    "tlogReplicas":"0"},
  "collection1":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{
      "shard1":{
        "range":"80000000-ffffffff",
        "state":"active",
        "replicas":{"core_node3":{
            "core":"collection1_shard1_replica_n1",
            "leader":"true",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:39155",
            "node_name":"127.0.0.1:39155_",
            "state":"active",
            "collection":"collection1",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}},
      "shard2":{
        "range":"0-7fffffff",
        "state":"active",
        "replicas":{
          "core_node4":{
            "core":"collection1_shard2_replica_n2",
            "leader":"true",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:33565",
            "node_name":"127.0.0.1:33565_",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node6":{
            "core":"collection1_shard2_replica_n5",
            "core_node_name":"core_node6",
            "base_url":"http://127.0.0.1:35541",
            "node_name":"127.0.0.1:35541_",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"}}
	at __randomizedtesting.SeedInfo.seed([BD7F9393C24DB6C9:352BAC496CB1DB31]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.ensureAllReplicasAreActive(AbstractFullDistribZkTestBase.java:2147)
	at org.apache.solr.cloud.HttpPartitionTest.testDoRecoveryOnRestart(HttpPartitionTest.java:200)
	at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:132)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:64)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:564)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1090)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1061)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:832)


FAILED:  org.apache.solr.cloud.HttpPartitionWithTlogReplicasTest.test

Error Message:
java.lang.AssertionError: Didn't see all replicas for shard shard1 in collDoRecoveryOnRestart come up within 130000 ms! ClusterState: {
  "control_collection":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{"core_node2":{
            "core":"control_collection_shard1_replica_n1",
            "leader":"true",
            "core_node_name":"core_node2",
            "base_url":"http://127.0.0.1:43693",
            "node_name":"127.0.0.1:43693_",
            "state":"active",
            "collection":"control_collection",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"},
  "collDoRecoveryOnRestart":{
    "pullReplicas":"0",
    "replicationFactor":"0",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"collDoRecoveryOnRestart_shard1_replica_t1",
            "leader":"true",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:43693",
            "node_name":"127.0.0.1:43693_",
            "state":"active",
            "collection":"collDoRecoveryOnRestart",
            "shard":"shard1",
            "type":"TLOG",
            "force_set_state":"false"},
          "core_node4":{
            "core":"collDoRecoveryOnRestart_shard1_replica_t2",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:35313",
            "node_name":"127.0.0.1:35313_",
            "state":"recovering",
            "collection":"collDoRecoveryOnRestart",
            "shard":"shard1",
            "type":"TLOG",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"0",
    "tlogReplicas":"2"},
  "collection1":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{
      "shard1":{
        "range":"80000000-ffffffff",
        "state":"active",
        "replicas":{"core_node4":{
            "core":"collection1_shard1_replica_t2",
            "leader":"true",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:35313",
            "node_name":"127.0.0.1:35313_",
            "state":"active",
            "collection":"collection1",
            "shard":"shard1",
            "type":"TLOG",
            "force_set_state":"false"}}},
      "shard2":{
        "range":"0-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"collection1_shard2_replica_t1",
            "leader":"true",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:38115",
            "node_name":"127.0.0.1:38115_",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"TLOG",
            "force_set_state":"false"},
          "core_node6":{
            "core":"collection1_shard2_replica_t5",
            "core_node_name":"core_node6",
            "base_url":"http://127.0.0.1:41745",
            "node_name":"127.0.0.1:41745_",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"TLOG",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"}}

Stack Trace:
java.lang.AssertionError: Didn't see all replicas for shard shard1 in collDoRecoveryOnRestart come up within 130000 ms! ClusterState: {
  "control_collection":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{"core_node2":{
            "core":"control_collection_shard1_replica_n1",
            "leader":"true",
            "core_node_name":"core_node2",
            "base_url":"http://127.0.0.1:43693",
            "node_name":"127.0.0.1:43693_",
            "state":"active",
            "collection":"control_collection",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"},
  "collDoRecoveryOnRestart":{
    "pullReplicas":"0",
    "replicationFactor":"0",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"collDoRecoveryOnRestart_shard1_replica_t1",
            "leader":"true",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:43693",
            "node_name":"127.0.0.1:43693_",
            "state":"active",
            "collection":"collDoRecoveryOnRestart",
            "shard":"shard1",
            "type":"TLOG",
            "force_set_state":"false"},
          "core_node4":{
            "core":"collDoRecoveryOnRestart_shard1_replica_t2",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:35313",
            "node_name":"127.0.0.1:35313_",
            "state":"recovering",
            "collection":"collDoRecoveryOnRestart",
            "shard":"shard1",
            "type":"TLOG",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"0",
    "tlogReplicas":"2"},
  "collection1":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{
      "shard1":{
        "range":"80000000-ffffffff",
        "state":"active",
        "replicas":{"core_node4":{
            "core":"collection1_shard1_replica_t2",
            "leader":"true",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:35313",
            "node_name":"127.0.0.1:35313_",
            "state":"active",
            "collection":"collection1",
            "shard":"shard1",
            "type":"TLOG",
            "force_set_state":"false"}}},
      "shard2":{
        "range":"0-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"collection1_shard2_replica_t1",
            "leader":"true",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:38115",
            "node_name":"127.0.0.1:38115_",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"TLOG",
            "force_set_state":"false"},
          "core_node6":{
            "core":"collection1_shard2_replica_t5",
            "core_node_name":"core_node6",
            "base_url":"http://127.0.0.1:41745",
            "node_name":"127.0.0.1:41745_",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"TLOG",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"}}
	at __randomizedtesting.SeedInfo.seed([BD7F9393C24DB6C9:352BAC496CB1DB31]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.ensureAllReplicasAreActive(AbstractFullDistribZkTestBase.java:2147)
	at org.apache.solr.cloud.HttpPartitionTest.testDoRecoveryOnRestart(HttpPartitionTest.java:200)
	at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:132)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:64)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:564)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1090)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1061)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:832)


FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test

Error Message:
java.lang.AssertionError: Didn't see all replicas for shard shard1 in c8n_1x3_lf come up within 90000 ms! ClusterState: {
  "control_collection":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{"core_node2":{
            "core":"control_collection_shard1_replica_n1",
            "leader":"true",
            "core_node_name":"core_node2",
            "base_url":"http://127.0.0.1:37521/xxia/bw",
            "node_name":"127.0.0.1:37521_xxia%2Fbw",
            "state":"active",
            "collection":"control_collection",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"},
  "c8n_1x3_lf":{
    "pullReplicas":"0",
    "replicationFactor":"3",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"c8n_1x3_lf_shard1_replica_n1",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:40165/xxia/bw",
            "node_name":"127.0.0.1:40165_xxia%2Fbw",
            "state":"recovering",
            "collection":"c8n_1x3_lf",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node5":{
            "core":"c8n_1x3_lf_shard1_replica_n2",
            "core_node_name":"core_node5",
            "base_url":"http://127.0.0.1:45275/xxia/bw",
            "node_name":"127.0.0.1:45275_xxia%2Fbw",
            "state":"recovering",
            "collection":"c8n_1x3_lf",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node6":{
            "core":"c8n_1x3_lf_shard1_replica_n4",
            "leader":"true",
            "core_node_name":"core_node6",
            "base_url":"http://127.0.0.1:37521/xxia/bw",
            "node_name":"127.0.0.1:37521_xxia%2Fbw",
            "state":"active",
            "collection":"c8n_1x3_lf",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"3",
    "tlogReplicas":"0"},
  "collection1":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{
      "shard1":{
        "range":"80000000-ffffffff",
        "state":"active",
        "replicas":{"core_node4":{
            "core":"collection1_shard1_replica_n2",
            "leader":"true",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:40165/xxia/bw",
            "node_name":"127.0.0.1:40165_xxia%2Fbw",
            "state":"active",
            "collection":"collection1",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}},
      "shard2":{
        "range":"0-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"collection1_shard2_replica_n1",
            "leader":"true",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:45275/xxia/bw",
            "node_name":"127.0.0.1:45275_xxia%2Fbw",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node6":{
            "core":"collection1_shard2_replica_n5",
            "core_node_name":"core_node6",
            "base_url":"http://127.0.0.1:43351/xxia/bw",
            "node_name":"127.0.0.1:43351_xxia%2Fbw",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"}}

Stack Trace:
java.lang.AssertionError: Didn't see all replicas for shard shard1 in c8n_1x3_lf come up within 90000 ms! ClusterState: {
  "control_collection":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{"core_node2":{
            "core":"control_collection_shard1_replica_n1",
            "leader":"true",
            "core_node_name":"core_node2",
            "base_url":"http://127.0.0.1:37521/xxia/bw",
            "node_name":"127.0.0.1:37521_xxia%2Fbw",
            "state":"active",
            "collection":"control_collection",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"},
  "c8n_1x3_lf":{
    "pullReplicas":"0",
    "replicationFactor":"3",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"c8n_1x3_lf_shard1_replica_n1",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:40165/xxia/bw",
            "node_name":"127.0.0.1:40165_xxia%2Fbw",
            "state":"recovering",
            "collection":"c8n_1x3_lf",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node5":{
            "core":"c8n_1x3_lf_shard1_replica_n2",
            "core_node_name":"core_node5",
            "base_url":"http://127.0.0.1:45275/xxia/bw",
            "node_name":"127.0.0.1:45275_xxia%2Fbw",
            "state":"recovering",
            "collection":"c8n_1x3_lf",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node6":{
            "core":"c8n_1x3_lf_shard1_replica_n4",
            "leader":"true",
            "core_node_name":"core_node6",
            "base_url":"http://127.0.0.1:37521/xxia/bw",
            "node_name":"127.0.0.1:37521_xxia%2Fbw",
            "state":"active",
            "collection":"c8n_1x3_lf",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"3",
    "tlogReplicas":"0"},
  "collection1":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{
      "shard1":{
        "range":"80000000-ffffffff",
        "state":"active",
        "replicas":{"core_node4":{
            "core":"collection1_shard1_replica_n2",
            "leader":"true",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:40165/xxia/bw",
            "node_name":"127.0.0.1:40165_xxia%2Fbw",
            "state":"active",
            "collection":"collection1",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}},
      "shard2":{
        "range":"0-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"collection1_shard2_replica_n1",
            "leader":"true",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:45275/xxia/bw",
            "node_name":"127.0.0.1:45275_xxia%2Fbw",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node6":{
            "core":"collection1_shard2_replica_n5",
            "core_node_name":"core_node6",
            "base_url":"http://127.0.0.1:43351/xxia/bw",
            "node_name":"127.0.0.1:43351_xxia%2Fbw",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"}}
	at __randomizedtesting.SeedInfo.seed([BD7F9393C24DB6C9:352BAC496CB1DB31]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.ensureAllReplicasAreActive(AbstractFullDistribZkTestBase.java:2147)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:101)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:58)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:64)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:564)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1090)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1061)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:832)


FAILED:  org.apache.solr.cloud.LeaderFailureAfterFreshStartTest.test

Error Message:
java.lang.AssertionError: timeout waiting to see all nodes active

Stack Trace:
java.lang.AssertionError: timeout waiting to see all nodes active
	at __randomizedtesting.SeedInfo.seed([BD7F9393C24DB6C9:352BAC496CB1DB31]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.solr.cloud.LeaderFailureAfterFreshStartTest.waitTillNodesActive(LeaderFailureAfterFreshStartTest.java:234)
	at org.apache.solr.cloud.LeaderFailureAfterFreshStartTest.restartNodes(LeaderFailureAfterFreshStartTest.java:174)
	at org.apache.solr.cloud.LeaderFailureAfterFreshStartTest.test(LeaderFailureAfterFreshStartTest.java:149)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:64)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:564)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1090)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1061)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:832)


FAILED:  org.apache.solr.cloud.LeaderVoteWaitTimeoutTest.testMostInSyncReplicasCanWinElection

Error Message:
java.lang.AssertionError: Timeout waiting for 1x3 collection
Timeout waiting to see state for collection=collection1 :DocCollection(collection1/26)={
  "pullReplicas":"0",
  "replicationFactor":"3",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node2":{
          "core":"collection1_shard1_replica_n1",
          "core_node_name":"core_node2",
          "base_url":"https://127.0.0.1:36027/solr",
          "node_name":"127.0.0.1:36027_solr",
          "state":"recovering",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node4":{
          "core":"collection1_shard1_replica_n3",
          "core_node_name":"core_node4",
          "base_url":"https://127.0.0.1:44263/solr",
          "node_name":"127.0.0.1:44263_solr",
          "state":"active",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node6":{
          "core":"collection1_shard1_replica_n5",
          "leader":"true",
          "core_node_name":"core_node6",
          "base_url":"https://127.0.0.1:33101/solr",
          "node_name":"127.0.0.1:33101_solr",
          "state":"active",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"3",
  "tlogReplicas":"0"}
Live Nodes: [127.0.0.1:33101_solr, 127.0.0.1:36027_solr, 127.0.0.1:44263_solr, 127.0.0.1:44317_solr]
Last available state: DocCollection(collection1/26)={
  "pullReplicas":"0",
  "replicationFactor":"3",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node2":{
          "core":"collection1_shard1_replica_n1",
          "core_node_name":"core_node2",
          "base_url":"https://127.0.0.1:36027/solr",
          "node_name":"127.0.0.1:36027_solr",
          "state":"recovering",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node4":{
          "core":"collection1_shard1_replica_n3",
          "core_node_name":"core_node4",
          "base_url":"https://127.0.0.1:44263/solr",
          "node_name":"127.0.0.1:44263_solr",
          "state":"active",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node6":{
          "core":"collection1_shard1_replica_n5",
          "leader":"true",
          "core_node_name":"core_node6",
          "base_url":"https://127.0.0.1:33101/solr",
          "node_name":"127.0.0.1:33101_solr",
          "state":"active",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"3",
  "tlogReplicas":"0"}

Stack Trace:
java.lang.AssertionError: Timeout waiting for 1x3 collection
Timeout waiting to see state for collection=collection1 :DocCollection(collection1/26)={
  "pullReplicas":"0",
  "replicationFactor":"3",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node2":{
          "core":"collection1_shard1_replica_n1",
          "core_node_name":"core_node2",
          "base_url":"https://127.0.0.1:36027/solr",
          "node_name":"127.0.0.1:36027_solr",
          "state":"recovering",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node4":{
          "core":"collection1_shard1_replica_n3",
          "core_node_name":"core_node4",
          "base_url":"https://127.0.0.1:44263/solr",
          "node_name":"127.0.0.1:44263_solr",
          "state":"active",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node6":{
          "core":"collection1_shard1_replica_n5",
          "leader":"true",
          "core_node_name":"core_node6",
          "base_url":"https://127.0.0.1:33101/solr",
          "node_name":"127.0.0.1:33101_solr",
          "state":"active",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"3",
  "tlogReplicas":"0"}
Live Nodes: [127.0.0.1:33101_solr, 127.0.0.1:36027_solr, 127.0.0.1:44263_solr, 127.0.0.1:44317_solr]
Last available state: DocCollection(collection1/26)={
  "pullReplicas":"0",
  "replicationFactor":"3",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node2":{
          "core":"collection1_shard1_replica_n1",
          "core_node_name":"core_node2",
          "base_url":"https://127.0.0.1:36027/solr",
          "node_name":"127.0.0.1:36027_solr",
          "state":"recovering",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node4":{
          "core":"collection1_shard1_replica_n3",
          "core_node_name":"core_node4",
          "base_url":"https://127.0.0.1:44263/solr",
          "node_name":"127.0.0.1:44263_solr",
          "state":"active",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node6":{
          "core":"collection1_shard1_replica_n5",
          "leader":"true",
          "core_node_name":"core_node6",
          "base_url":"https://127.0.0.1:33101/solr",
          "node_name":"127.0.0.1:33101_solr",
          "state":"active",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"3",
  "tlogReplicas":"0"}
	at __randomizedtesting.SeedInfo.seed([BD7F9393C24DB6C9:15638F29000D82E3]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.solr.cloud.SolrCloudTestCase.waitForState(SolrCloudTestCase.java:326)
	at org.apache.solr.cloud.SolrCloudTestCase.waitForState(SolrCloudTestCase.java:303)
	at org.apache.solr.cloud.LeaderVoteWaitTimeoutTest.testMostInSyncReplicasCanWinElection(LeaderVoteWaitTimeoutTest.java:253)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:64)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:564)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:832)


FAILED:  org.apache.solr.cloud.MigrateRouteKeyTest.multipleShardMigrateTest

Error Message:
java.lang.AssertionError: DocCount on target collection does not match expected:<19> but was:<16>

Stack Trace:
java.lang.AssertionError: DocCount on target collection does not match expected:<19> but was:<16>
	at __randomizedtesting.SeedInfo.seed([BD7F9393C24DB6C9:A16F89E9E38D0628]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.junit.Assert.failNotEquals(Assert.java:835)
	at org.junit.Assert.assertEquals(Assert.java:647)
	at org.apache.solr.cloud.MigrateRouteKeyTest.multipleShardMigrateTest(MigrateRouteKeyTest.java:159)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:64)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:564)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.Statemen

[...truncated too long message...]

dOops -XX:+UseParallelGC" -Ptests.seed=BD7F9393C24DB6C9 -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=UTF-8

  - org.apache.solr.cloud.TestCloudRecovery2.test (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.TestCloudRecovery2.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.TestCloudRecovery2" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseParallelGC" -Ptests.seed=BD7F9393C24DB6C9 -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=UTF-8

  - org.apache.solr.cloud.MissingSegmentRecoveryTest.testLeaderRecovery (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.MissingSegmentRecoveryTest.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.MissingSegmentRecoveryTest" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseParallelGC" -Ptests.seed=BD7F9393C24DB6C9 -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=UTF-8

  - org.apache.solr.cloud.LeaderVoteWaitTimeoutTest.testMostInSyncReplicasCanWinElection (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.LeaderVoteWaitTimeoutTest.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.LeaderVoteWaitTimeoutTest" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseParallelGC" -Ptests.seed=BD7F9393C24DB6C9 -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=UTF-8

  - org.apache.solr.cloud.UnloadDistributedZkTest.test (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.UnloadDistributedZkTest.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.UnloadDistributedZkTest" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseParallelGC" -Ptests.seed=BD7F9393C24DB6C9 -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=UTF-8

  - org.apache.solr.cloud.HttpPartitionTest.test (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.HttpPartitionTest.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.HttpPartitionTest" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseParallelGC" -Ptests.seed=BD7F9393C24DB6C9 -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=UTF-8

  - org.apache.solr.cloud.SyncSliceTest.test (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.SyncSliceTest.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.SyncSliceTest" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseParallelGC" -Ptests.seed=BD7F9393C24DB6C9 -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=UTF-8

  - org.apache.solr.cloud.TestCloudRecovery.corruptedLogTest (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.TestCloudRecovery.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.TestCloudRecovery" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseParallelGC" -Ptests.seed=BD7F9393C24DB6C9 -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=UTF-8

  - org.apache.solr.cloud.MigrateRouteKeyTest.multipleShardMigrateTest (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.MigrateRouteKeyTest.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.MigrateRouteKeyTest" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseParallelGC" -Ptests.seed=BD7F9393C24DB6C9 -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=UTF-8

  - org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.RecoveryAfterSoftCommitTest.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.RecoveryAfterSoftCommitTest" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseParallelGC" -Ptests.seed=BD7F9393C24DB6C9 -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=UTF-8

  - org.apache.solr.cloud.DeleteReplicaTest.raceConditionOnDeleteAndRegisterReplica (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.DeleteReplicaTest.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.DeleteReplicaTest" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseParallelGC" -Ptests.seed=BD7F9393C24DB6C9 -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=UTF-8

  - org.apache.solr.cloud.TestLeaderElectionWithEmptyReplica.test (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.TestLeaderElectionWithEmptyReplica.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.TestLeaderElectionWithEmptyReplica" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseParallelGC" -Ptests.seed=BD7F9393C24DB6C9 -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=UTF-8

  - org.apache.solr.core.snapshots.TestSolrCloudSnapshots.testSnapshots (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.core.snapshots.TestSolrCloudSnapshots.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.core.snapshots.TestSolrCloudSnapshots" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseParallelGC" -Ptests.seed=BD7F9393C24DB6C9 -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=UTF-8

  - org.apache.solr.cloud.RecoveryZkTest.test (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.RecoveryZkTest.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.RecoveryZkTest" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseParallelGC" -Ptests.seed=BD7F9393C24DB6C9 -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=UTF-8


Deprecated Gradle features were used in this build, making it incompatible with Gradle 7.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/6.6.1/userguide/command_line_interface.html#sec:command_line_warnings

BUILD SUCCESSFUL in 59m 33s
854 actionable tasks: 854 executed
Build step 'Invoke Gradle script' changed build result to SUCCESS
Archiving artifacts
java.lang.InterruptedException: no matches found within 10000
	at hudson.FilePath$ValidateAntFileMask.hasMatch(FilePath.java:2875)
	at hudson.FilePath$ValidateAntFileMask.invoke(FilePath.java:2754)
	at hudson.FilePath$ValidateAntFileMask.invoke(FilePath.java:2735)
	at hudson.FilePath.act(FilePath.java:1076)
	at hudson.FilePath.act(FilePath.java:1059)
	at hudson.FilePath.validateAntFileMask(FilePath.java:2733)
	at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:270)
	at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:80)
	at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
	at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:804)
	at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:753)
	at hudson.model.Build$BuildExecution.post2(Build.java:177)
	at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:698)
	at hudson.model.Run.execute(Run.java:1919)
	at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
	at hudson.model.ResourceController.execute(ResourceController.java:97)
	at hudson.model.Executor.run(Executor.java:428)
No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/*_pid*.log". Configuration error?
[Java] Sleeping for 5 seconds due to JENKINS-32191...
[Java] Parsing console log (workspace: '/home/jenkins/workspace/Lucene-Solr-master-Linux')
[Java] Successfully parsed console log
[Java] -> found 0 issues (skipped 0 duplicates)
[Java] Skipping post processing
[Java] No filter has been set, publishing all 0 issues
[Java] Using reference build 'Lucene-Solr-master-Linux #28357' to compute new, fixed, and outstanding issues
[Java] Issues delta (vs. reference build): outstanding: 0, new: 0, fixed: 0
[Java] No quality gates have been set - skipping
[Java] Health report is disabled - skipping
[Java] Created analysis result for 0 issues (found 0 new issues, fixed 0 issues)
[Java] Attaching ResultAction with ID 'java' to run 'Lucene-Solr-master-Linux #28358'.
[Checks API] No suitable checks publisher found.
Recording test results
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
[Checks API] No suitable checks publisher found.
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)

[JENKINS] Lucene-Solr-master-Linux (64bit/jdk-15) - Build # 28365 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Linux/28365/
Java: 64bit/jdk-15 -XX:+UseCompressedOops -XX:+UseSerialGC

1 tests failed.
FAILED:  org.apache.solr.cloud.TestPullReplicaErrorHandling.testCantConnectToLeader

Error Message:
java.lang.AssertionError: Didn't get expected doc count. Expected: 10, Found: 0

Stack Trace:
java.lang.AssertionError: Didn't get expected doc count. Expected: 10, Found: 0
	at __randomizedtesting.SeedInfo.seed([FFDF285C4CA8973A:71D3482C39B01310]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.solr.cloud.TestPullReplicaErrorHandling.assertNumDocs(TestPullReplicaErrorHandling.java:238)
	at org.apache.solr.cloud.TestPullReplicaErrorHandling.assertNumDocs(TestPullReplicaErrorHandling.java:243)
	at org.apache.solr.cloud.TestPullReplicaErrorHandling.testCantConnectToLeader(TestPullReplicaErrorHandling.java:183)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:64)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:564)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:832)




Build Log:
[...truncated 3483 lines...]
ERROR: The following test(s) have failed:
  - org.apache.solr.cloud.TestPullReplicaErrorHandling.testCantConnectToLeader (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.TestPullReplicaErrorHandling.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.TestPullReplicaErrorHandling" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:+UseCompressedOops -XX:+UseSerialGC" -Ptests.seed=FFDF285C4CA8973A -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=US-ASCII


Deprecated Gradle features were used in this build, making it incompatible with Gradle 7.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/6.6.1/userguide/command_line_interface.html#sec:command_line_warnings

BUILD SUCCESSFUL in 40m 44s
854 actionable tasks: 854 executed
Build step 'Invoke Gradle script' changed build result to SUCCESS
Archiving artifacts
java.lang.InterruptedException: no matches found within 10000
	at hudson.FilePath$ValidateAntFileMask.hasMatch(FilePath.java:2875)
	at hudson.FilePath$ValidateAntFileMask.invoke(FilePath.java:2754)
	at hudson.FilePath$ValidateAntFileMask.invoke(FilePath.java:2735)
	at hudson.FilePath.act(FilePath.java:1076)
	at hudson.FilePath.act(FilePath.java:1059)
	at hudson.FilePath.validateAntFileMask(FilePath.java:2733)
	at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:270)
	at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:80)
	at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
	at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:804)
	at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:753)
	at hudson.model.Build$BuildExecution.post2(Build.java:177)
	at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:698)
	at hudson.model.Run.execute(Run.java:1919)
	at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
	at hudson.model.ResourceController.execute(ResourceController.java:97)
	at hudson.model.Executor.run(Executor.java:428)
No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/*_pid*.log". Configuration error?
[Java] Sleeping for 5 seconds due to JENKINS-32191...
[Java] Parsing console log (workspace: '/home/jenkins/workspace/Lucene-Solr-master-Linux')
[Java] Successfully parsed console log
[Java] -> found 0 issues (skipped 0 duplicates)
[Java] Skipping post processing
[Java] No filter has been set, publishing all 0 issues
[Java] Using reference build 'Lucene-Solr-master-Linux #28364' to compute new, fixed, and outstanding issues
[Java] Issues delta (vs. reference build): outstanding: 0, new: 0, fixed: 0
[Java] No quality gates have been set - skipping
[Java] Health report is disabled - skipping
[Java] Created analysis result for 0 issues (found 0 new issues, fixed 0 issues)
[Java] Attaching ResultAction with ID 'java' to run 'Lucene-Solr-master-Linux #28365'.
[Checks API] No suitable checks publisher found.
Recording test results
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
[Checks API] No suitable checks publisher found.
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)

[JENKINS] Lucene-Solr-master-Linux (64bit/jdk-15) - Build # 28364 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Linux/28364/
Java: 64bit/jdk-15 -XX:-UseCompressedOops -XX:+UseZGC

52 tests failed.
FAILED:  org.apache.solr.cloud.AssignBackwardCompatibilityTest.test

Error Message:
java.lang.RuntimeException: Failed while waiting for active collection
Timeout waiting to see state for collection=collection1 :DocCollection(collection1/55)={
  "pullReplicas":"0",
  "replicationFactor":"4",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node8":{
          "core":"collection1_shard1_replica_n7",
          "leader":"true",
          "core_node_name":"core_node8",
          "base_url":"https://127.0.0.1:42703/solr",
          "node_name":"127.0.0.1:42703_solr",
          "state":"active",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node10":{
          "core":"collection1_shard1_replica_n9",
          "core_node_name":"core_node10",
          "base_url":"https://127.0.0.1:37793/solr",
          "node_name":"127.0.0.1:37793_solr",
          "state":"active",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node12":{
          "core":"collection1_shard1_replica_n11",
          "core_node_name":"core_node12",
          "base_url":"https://127.0.0.1:42703/solr",
          "node_name":"127.0.0.1:42703_solr",
          "state":"active",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node14":{
          "core":"collection1_shard1_replica_n13",
          "core_node_name":"core_node14",
          "base_url":"https://127.0.0.1:45725/solr",
          "node_name":"127.0.0.1:45725_solr",
          "state":"recovering",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"4",
  "tlogReplicas":"0"}
Live Nodes: [127.0.0.1:37793_solr, 127.0.0.1:42703_solr, 127.0.0.1:42879_solr, 127.0.0.1:45725_solr]
Last available state: DocCollection(collection1/55)={
  "pullReplicas":"0",
  "replicationFactor":"4",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node8":{
          "core":"collection1_shard1_replica_n7",
          "leader":"true",
          "core_node_name":"core_node8",
          "base_url":"https://127.0.0.1:42703/solr",
          "node_name":"127.0.0.1:42703_solr",
          "state":"active",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node10":{
          "core":"collection1_shard1_replica_n9",
          "core_node_name":"core_node10",
          "base_url":"https://127.0.0.1:37793/solr",
          "node_name":"127.0.0.1:37793_solr",
          "state":"active",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node12":{
          "core":"collection1_shard1_replica_n11",
          "core_node_name":"core_node12",
          "base_url":"https://127.0.0.1:42703/solr",
          "node_name":"127.0.0.1:42703_solr",
          "state":"active",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node14":{
          "core":"collection1_shard1_replica_n13",
          "core_node_name":"core_node14",
          "base_url":"https://127.0.0.1:45725/solr",
          "node_name":"127.0.0.1:45725_solr",
          "state":"recovering",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"4",
  "tlogReplicas":"0"}

Stack Trace:
java.lang.RuntimeException: Failed while waiting for active collection
Timeout waiting to see state for collection=collection1 :DocCollection(collection1/55)={
  "pullReplicas":"0",
  "replicationFactor":"4",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node8":{
          "core":"collection1_shard1_replica_n7",
          "leader":"true",
          "core_node_name":"core_node8",
          "base_url":"https://127.0.0.1:42703/solr",
          "node_name":"127.0.0.1:42703_solr",
          "state":"active",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node10":{
          "core":"collection1_shard1_replica_n9",
          "core_node_name":"core_node10",
          "base_url":"https://127.0.0.1:37793/solr",
          "node_name":"127.0.0.1:37793_solr",
          "state":"active",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node12":{
          "core":"collection1_shard1_replica_n11",
          "core_node_name":"core_node12",
          "base_url":"https://127.0.0.1:42703/solr",
          "node_name":"127.0.0.1:42703_solr",
          "state":"active",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node14":{
          "core":"collection1_shard1_replica_n13",
          "core_node_name":"core_node14",
          "base_url":"https://127.0.0.1:45725/solr",
          "node_name":"127.0.0.1:45725_solr",
          "state":"recovering",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"4",
  "tlogReplicas":"0"}
Live Nodes: [127.0.0.1:37793_solr, 127.0.0.1:42703_solr, 127.0.0.1:42879_solr, 127.0.0.1:45725_solr]
Last available state: DocCollection(collection1/55)={
  "pullReplicas":"0",
  "replicationFactor":"4",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node8":{
          "core":"collection1_shard1_replica_n7",
          "leader":"true",
          "core_node_name":"core_node8",
          "base_url":"https://127.0.0.1:42703/solr",
          "node_name":"127.0.0.1:42703_solr",
          "state":"active",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node10":{
          "core":"collection1_shard1_replica_n9",
          "core_node_name":"core_node10",
          "base_url":"https://127.0.0.1:37793/solr",
          "node_name":"127.0.0.1:37793_solr",
          "state":"active",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node12":{
          "core":"collection1_shard1_replica_n11",
          "core_node_name":"core_node12",
          "base_url":"https://127.0.0.1:42703/solr",
          "node_name":"127.0.0.1:42703_solr",
          "state":"active",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node14":{
          "core":"collection1_shard1_replica_n13",
          "core_node_name":"core_node14",
          "base_url":"https://127.0.0.1:45725/solr",
          "node_name":"127.0.0.1:45725_solr",
          "state":"recovering",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"4",
  "tlogReplicas":"0"}
	at __randomizedtesting.SeedInfo.seed([BEB7C845F8FAECE:83BF435EF173C336]:0)
	at org.apache.solr.cloud.MiniSolrCloudCluster.waitForActiveCollection(MiniSolrCloudCluster.java:774)
	at org.apache.solr.cloud.MiniSolrCloudCluster.waitForActiveCollection(MiniSolrCloudCluster.java:780)
	at org.apache.solr.cloud.AssignBackwardCompatibilityTest.test(AssignBackwardCompatibilityTest.java:95)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:64)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:564)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:832)


FAILED:  org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test

Error Message:
java.lang.AssertionError: There are still nodes recovering - waited for 600SECONDS

Stack Trace:
java.lang.AssertionError: There are still nodes recovering - waited for 600SECONDS
	at __randomizedtesting.SeedInfo.seed([BEB7C845F8FAECE:83BF435EF173C336]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:197)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:150)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:1063)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForThingsToLevelOut(AbstractFullDistribZkTestBase.java:1621)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForThingsToLevelOut(AbstractFullDistribZkTestBase.java:1613)
	at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test(ChaosMonkeyNothingIsSafeTest.java:236)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:64)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:564)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1090)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1061)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:832)


FAILED:  org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest.test

Error Message:
java.lang.AssertionError: There are still nodes recovering - waited for 180SECONDS

Stack Trace:
java.lang.AssertionError: There are still nodes recovering - waited for 180SECONDS
	at __randomizedtesting.SeedInfo.seed([BEB7C845F8FAECE:83BF435EF173C336]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:197)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:150)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:1063)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForThingsToLevelOut(AbstractFullDistribZkTestBase.java:1621)
	at org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest.test(ChaosMonkeySafeLeaderWithPullReplicasTest.java:198)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:64)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:564)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1090)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1061)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:832)


FAILED:  org.apache.solr.cloud.api.collections.TestHdfsCloudBackupRestore.test

Error Message:
java.lang.AssertionError

Stack Trace:
java.lang.AssertionError
	at __randomizedtesting.SeedInfo.seed([BEB7C845F8FAECE:83BF435EF173C336]:0)
	at org.junit.Assert.fail(Assert.java:87)
	at org.junit.Assert.assertTrue(Assert.java:42)
	at org.junit.Assert.assertTrue(Assert.java:53)
	at org.apache.solr.cloud.api.collections.AbstractCloudBackupRestoreTestCase.test(AbstractCloudBackupRestoreTestCase.java:148)
	at org.apache.solr.cloud.api.collections.TestHdfsCloudBackupRestore.test(TestHdfsCloudBackupRestore.java:225)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:64)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:564)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:832)


FAILED:  org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest.test

Error Message:
java.util.concurrent.TimeoutException: Cluster state still in flux after 30 retry intervals.

Stack Trace:
java.util.concurrent.TimeoutException: Cluster state still in flux after 30 retry intervals.
	at __randomizedtesting.SeedInfo.seed([BEB7C845F8FAECE:83BF435EF173C336]:0)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForThingsToLevelOut(AbstractFullDistribZkTestBase.java:1643)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForThingsToLevelOut(AbstractFullDistribZkTestBase.java:1613)
	at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest.test(ChaosMonkeyNothingIsSafeWithPullReplicasTest.java:253)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:64)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:564)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1090)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1061)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:832)


FAILED:  org.apache.solr.cloud.MoveReplicaTest.test

Error Message:
java.lang.AssertionError

Stack Trace:
java.lang.AssertionError
	at __randomizedtesting.SeedInfo.seed([BEB7C845F8FAECE:83BF435EF173C336]:0)
	at org.junit.Assert.fail(Assert.java:87)
	at org.junit.Assert.assertTrue(Assert.java:42)
	at org.junit.Assert.assertTrue(Assert.java:53)
	at org.apache.solr.cloud.MoveReplicaTest.test(MoveReplicaTest.java:150)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:64)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:564)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:832)


FAILED:  org.apache.solr.cloud.BasicDistributedZk2Test.test

Error Message:
java.lang.AssertionError: There are still nodes recovering - waited for 330SECONDS

Stack Trace:
java.lang.AssertionError: There are still nodes recovering - waited for 330SECONDS
	at __randomizedtesting.SeedInfo.seed([BEB7C845F8FAECE:83BF435EF173C336]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:197)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:144)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:139)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:1051)
	at org.apache.solr.cloud.BasicDistributedZk2Test.brindDownShardIndexSomeDocsAndRecover(BasicDistributedZk2Test.java:362)
	at org.apache.solr.cloud.BasicDistributedZk2Test.test(BasicDistributedZk2Test.java:103)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:64)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:564)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1090)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1061)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:832)


FAILED:  org.apache.solr.cloud.DeleteReplicaTest.raceConditionOnDeleteAndRegisterReplica

Error Message:
java.lang.RuntimeException: Failed while waiting for active collection
Timeout waiting to see state for collection=raceDeleteReplicaCollection :DocCollection(raceDeleteReplicaCollection/16)={
  "pullReplicas":"0",
  "replicationFactor":"2",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node4":{
          "core":"raceDeleteReplicaCollection_shard1_replica_n2",
          "core_node_name":"core_node4",
          "base_url":"https://127.0.0.1:40415/solr",
          "node_name":"127.0.0.1:40415_solr",
          "state":"recovering",
          "collection":"raceDeleteReplicaCollection",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node6":{
          "core":"raceDeleteReplicaCollection_shard1_replica_n5",
          "leader":"true",
          "core_node_name":"core_node6",
          "base_url":"https://127.0.0.1:34371/solr",
          "node_name":"127.0.0.1:34371_solr",
          "state":"active",
          "collection":"raceDeleteReplicaCollection",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"2",
  "tlogReplicas":"0"}
Live Nodes: [127.0.0.1:34371_solr, 127.0.0.1:40415_solr, 127.0.0.1:45363_solr, 127.0.0.1:45787_solr]
Last available state: DocCollection(raceDeleteReplicaCollection/16)={
  "pullReplicas":"0",
  "replicationFactor":"2",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node4":{
          "core":"raceDeleteReplicaCollection_shard1_replica_n2",
          "core_node_name":"core_node4",
          "base_url":"https://127.0.0.1:40415/solr",
          "node_name":"127.0.0.1:40415_solr",
          "state":"recovering",
          "collection":"raceDeleteReplicaCollection",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node6":{
          "core":"raceDeleteReplicaCollection_shard1_replica_n5",
          "leader":"true",
          "core_node_name":"core_node6",
          "base_url":"https://127.0.0.1:34371/solr",
          "node_name":"127.0.0.1:34371_solr",
          "state":"active",
          "collection":"raceDeleteReplicaCollection",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"2",
  "tlogReplicas":"0"}

Stack Trace:
java.lang.RuntimeException: Failed while waiting for active collection
Timeout waiting to see state for collection=raceDeleteReplicaCollection :DocCollection(raceDeleteReplicaCollection/16)={
  "pullReplicas":"0",
  "replicationFactor":"2",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node4":{
          "core":"raceDeleteReplicaCollection_shard1_replica_n2",
          "core_node_name":"core_node4",
          "base_url":"https://127.0.0.1:40415/solr",
          "node_name":"127.0.0.1:40415_solr",
          "state":"recovering",
          "collection":"raceDeleteReplicaCollection",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node6":{
          "core":"raceDeleteReplicaCollection_shard1_replica_n5",
          "leader":"true",
          "core_node_name":"core_node6",
          "base_url":"https://127.0.0.1:34371/solr",
          "node_name":"127.0.0.1:34371_solr",
          "state":"active",
          "collection":"raceDeleteReplicaCollection",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"2",
  "tlogReplicas":"0"}
Live Nodes: [127.0.0.1:34371_solr, 127.0.0.1:40415_solr, 127.0.0.1:45363_solr, 127.0.0.1:45787_solr]
Last available state: DocCollection(raceDeleteReplicaCollection/16)={
  "pullReplicas":"0",
  "replicationFactor":"2",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node4":{
          "core":"raceDeleteReplicaCollection_shard1_replica_n2",
          "core_node_name":"core_node4",
          "base_url":"https://127.0.0.1:40415/solr",
          "node_name":"127.0.0.1:40415_solr",
          "state":"recovering",
          "collection":"raceDeleteReplicaCollection",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node6":{
          "core":"raceDeleteReplicaCollection_shard1_replica_n5",
          "leader":"true",
          "core_node_name":"core_node6",
          "base_url":"https://127.0.0.1:34371/solr",
          "node_name":"127.0.0.1:34371_solr",
          "state":"active",
          "collection":"raceDeleteReplicaCollection",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"2",
  "tlogReplicas":"0"}
	at __randomizedtesting.SeedInfo.seed([BEB7C845F8FAECE:61FD1D54377DE404]:0)
	at org.apache.solr.cloud.MiniSolrCloudCluster.waitForActiveCollection(MiniSolrCloudCluster.java:774)
	at org.apache.solr.cloud.MiniSolrCloudCluster.waitForActiveCollection(MiniSolrCloudCluster.java:780)
	at org.apache.solr.cloud.DeleteReplicaTest.raceConditionOnDeleteAndRegisterReplica(DeleteReplicaTest.java:385)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:64)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:564)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:832)


FAILED:  org.apache.solr.cloud.HttpPartitionTest.test

Error Message:
java.lang.AssertionError: Didn't see all replicas for shard shard1 in collDoRecoveryOnRestart come up within 130000 ms! ClusterState: {
  "control_collection":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{"core_node2":{
            "core":"control_collection_shard1_replica_n1",
            "leader":"true",
            "core_node_name":"core_node2",
            "base_url":"http://127.0.0.1:33385/ulm/lt",
            "node_name":"127.0.0.1:33385_ulm%2Flt",
            "state":"active",
            "collection":"control_collection",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"},
  "collDoRecoveryOnRestart":{
    "pullReplicas":"0",
    "replicationFactor":"2",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"collDoRecoveryOnRestart_shard1_replica_n1",
            "leader":"true",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:43433/ulm/lt",
            "node_name":"127.0.0.1:43433_ulm%2Flt",
            "state":"active",
            "collection":"collDoRecoveryOnRestart",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node4":{
            "core":"collDoRecoveryOnRestart_shard1_replica_n2",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:33385/ulm/lt",
            "node_name":"127.0.0.1:33385_ulm%2Flt",
            "state":"recovering",
            "collection":"collDoRecoveryOnRestart",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"2",
    "tlogReplicas":"0"},
  "collection1":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{
      "shard1":{
        "range":"80000000-ffffffff",
        "state":"active",
        "replicas":{"core_node4":{
            "core":"collection1_shard1_replica_n2",
            "leader":"true",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:36711/ulm/lt",
            "node_name":"127.0.0.1:36711_ulm%2Flt",
            "state":"active",
            "collection":"collection1",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}},
      "shard2":{
        "range":"0-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"collection1_shard2_replica_n1",
            "leader":"true",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:43433/ulm/lt",
            "node_name":"127.0.0.1:43433_ulm%2Flt",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node6":{
            "core":"collection1_shard2_replica_n5",
            "core_node_name":"core_node6",
            "base_url":"http://127.0.0.1:34541/ulm/lt",
            "node_name":"127.0.0.1:34541_ulm%2Flt",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"}}

Stack Trace:
java.lang.AssertionError: Didn't see all replicas for shard shard1 in collDoRecoveryOnRestart come up within 130000 ms! ClusterState: {
  "control_collection":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{"core_node2":{
            "core":"control_collection_shard1_replica_n1",
            "leader":"true",
            "core_node_name":"core_node2",
            "base_url":"http://127.0.0.1:33385/ulm/lt",
            "node_name":"127.0.0.1:33385_ulm%2Flt",
            "state":"active",
            "collection":"control_collection",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"},
  "collDoRecoveryOnRestart":{
    "pullReplicas":"0",
    "replicationFactor":"2",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"collDoRecoveryOnRestart_shard1_replica_n1",
            "leader":"true",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:43433/ulm/lt",
            "node_name":"127.0.0.1:43433_ulm%2Flt",
            "state":"active",
            "collection":"collDoRecoveryOnRestart",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node4":{
            "core":"collDoRecoveryOnRestart_shard1_replica_n2",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:33385/ulm/lt",
            "node_name":"127.0.0.1:33385_ulm%2Flt",
            "state":"recovering",
            "collection":"collDoRecoveryOnRestart",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"2",
    "tlogReplicas":"0"},
  "collection1":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{
      "shard1":{
        "range":"80000000-ffffffff",
        "state":"active",
        "replicas":{"core_node4":{
            "core":"collection1_shard1_replica_n2",
            "leader":"true",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:36711/ulm/lt",
            "node_name":"127.0.0.1:36711_ulm%2Flt",
            "state":"active",
            "collection":"collection1",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}},
      "shard2":{
        "range":"0-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"collection1_shard2_replica_n1",
            "leader":"true",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:43433/ulm/lt",
            "node_name":"127.0.0.1:43433_ulm%2Flt",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node6":{
            "core":"collection1_shard2_replica_n5",
            "core_node_name":"core_node6",
            "base_url":"http://127.0.0.1:34541/ulm/lt",
            "node_name":"127.0.0.1:34541_ulm%2Flt",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"}}
	at __randomizedtesting.SeedInfo.seed([BEB7C845F8FAECE:83BF435EF173C336]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.ensureAllReplicasAreActive(AbstractFullDistribZkTestBase.java:2147)
	at org.apache.solr.cloud.HttpPartitionTest.testDoRecoveryOnRestart(HttpPartitionTest.java:200)
	at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:132)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:64)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:564)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1090)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1061)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:832)


FAILED:  org.apache.solr.cloud.HttpPartitionWithTlogReplicasTest.test

Error Message:
java.lang.AssertionError: Didn't see all replicas for shard shard1 in collDoRecoveryOnRestart come up within 130000 ms! ClusterState: {
  "control_collection":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{"core_node2":{
            "core":"control_collection_shard1_replica_n1",
            "leader":"true",
            "core_node_name":"core_node2",
            "base_url":"http://127.0.0.1:34351",
            "node_name":"127.0.0.1:34351_",
            "state":"active",
            "collection":"control_collection",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"},
  "collDoRecoveryOnRestart":{
    "pullReplicas":"0",
    "replicationFactor":"0",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"collDoRecoveryOnRestart_shard1_replica_t1",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:32979",
            "node_name":"127.0.0.1:32979_",
            "state":"recovering",
            "collection":"collDoRecoveryOnRestart",
            "shard":"shard1",
            "type":"TLOG",
            "force_set_state":"false"},
          "core_node4":{
            "core":"collDoRecoveryOnRestart_shard1_replica_t2",
            "leader":"true",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:34351",
            "node_name":"127.0.0.1:34351_",
            "state":"active",
            "collection":"collDoRecoveryOnRestart",
            "shard":"shard1",
            "type":"TLOG",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"0",
    "tlogReplicas":"2"},
  "collection1":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{
      "shard1":{
        "range":"80000000-ffffffff",
        "state":"active",
        "replicas":{"core_node4":{
            "core":"collection1_shard1_replica_t2",
            "leader":"true",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:32979",
            "node_name":"127.0.0.1:32979_",
            "state":"active",
            "collection":"collection1",
            "shard":"shard1",
            "type":"TLOG",
            "force_set_state":"false"}}},
      "shard2":{
        "range":"0-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"collection1_shard2_replica_t1",
            "leader":"true",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:42781",
            "node_name":"127.0.0.1:42781_",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"TLOG",
            "force_set_state":"false"},
          "core_node6":{
            "core":"collection1_shard2_replica_t5",
            "core_node_name":"core_node6",
            "base_url":"http://127.0.0.1:38803",
            "node_name":"127.0.0.1:38803_",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"TLOG",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"}}

Stack Trace:
java.lang.AssertionError: Didn't see all replicas for shard shard1 in collDoRecoveryOnRestart come up within 130000 ms! ClusterState: {
  "control_collection":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{"core_node2":{
            "core":"control_collection_shard1_replica_n1",
            "leader":"true",
            "core_node_name":"core_node2",
            "base_url":"http://127.0.0.1:34351",
            "node_name":"127.0.0.1:34351_",
            "state":"active",
            "collection":"control_collection",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"},
  "collDoRecoveryOnRestart":{
    "pullReplicas":"0",
    "replicationFactor":"0",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"collDoRecoveryOnRestart_shard1_replica_t1",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:32979",
            "node_name":"127.0.0.1:32979_",
            "state":"recovering",
            "collection":"collDoRecoveryOnRestart",
            "shard":"shard1",
            "type":"TLOG",
            "force_set_state":"false"},
          "core_node4":{
            "core":"collDoRecoveryOnRestart_shard1_replica_t2",
            "leader":"true",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:34351",
            "node_name":"127.0.0.1:34351_",
            "state":"active",
            "collection":"collDoRecoveryOnRestart",
            "shard":"shard1",
            "type":"TLOG",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"0",
    "tlogReplicas":"2"},
  "collection1":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{
      "shard1":{
        "range":"80000000-ffffffff",
        "state":"active",
        "replicas":{"core_node4":{
            "core":"collection1_shard1_replica_t2",
            "leader":"true",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:32979",
            "node_name":"127.0.0.1:32979_",
            "state":"active",
            "collection":"collection1",
            "shard":"shard1",
            "type":"TLOG",
            "force_set_state":"false"}}},
      "shard2":{
        "range":"0-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"collection1_shard2_replica_t1",
            "leader":"true",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:42781",
            "node_name":"127.0.0.1:42781_",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"TLOG",
            "force_set_state":"false"},
          "core_node6":{
            "core":"collection1_shard2_replica_t5",
            "core_node_name":"core_node6",
            "base_url":"http://127.0.0.1:38803",
            "node_name":"127.0.0.1:38803_",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"TLOG",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"}}
	at __randomizedtesting.SeedInfo.seed([BEB7C845F8FAECE:83BF435EF173C336]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.ensureAllReplicasAreActive(AbstractFullDistribZkTestBase.java:2147)
	at org.apache.solr.cloud.HttpPartitionTest.testDoRecoveryOnRestart(HttpPartitionTest.java:200)
	at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:132)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:64)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:564)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1090)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1061)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:832)


FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test

Error Message:
java.lang.AssertionError: Didn't see all replicas for shard shard1 in c8n_1x3_lf come up within 90000 ms! ClusterState: {
  "control_collection":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{"core_node2":{
            "core":"control_collection_shard1_replica_n1",
            "leader":"true",
            "core_node_name":"core_node2",
            "base_url":"http://127.0.0.1:44041/g_/p",
            "node_name":"127.0.0.1:44041_g_%2Fp",
            "state":"active",
            "collection":"control_collection",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"},
  "c8n_1x3_lf":{
    "pullReplicas":"0",
    "replicationFactor":"3",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"c8n_1x3_lf_shard1_replica_n1",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:44041/g_/p",
            "node_name":"127.0.0.1:44041_g_%2Fp",
            "state":"recovering",
            "collection":"c8n_1x3_lf",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node5":{
            "core":"c8n_1x3_lf_shard1_replica_n2",
            "leader":"true",
            "core_node_name":"core_node5",
            "base_url":"http://127.0.0.1:42559/g_/p",
            "node_name":"127.0.0.1:42559_g_%2Fp",
            "state":"active",
            "collection":"c8n_1x3_lf",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node6":{
            "core":"c8n_1x3_lf_shard1_replica_n4",
            "core_node_name":"core_node6",
            "base_url":"http://127.0.0.1:45159/g_/p",
            "node_name":"127.0.0.1:45159_g_%2Fp",
            "state":"recovering",
            "collection":"c8n_1x3_lf",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"3",
    "tlogReplicas":"0"},
  "collection1":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{
      "shard1":{
        "range":"80000000-ffffffff",
        "state":"active",
        "replicas":{"core_node3":{
            "core":"collection1_shard1_replica_n1",
            "leader":"true",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:45159/g_/p",
            "node_name":"127.0.0.1:45159_g_%2Fp",
            "state":"active",
            "collection":"collection1",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}},
      "shard2":{
        "range":"0-7fffffff",
        "state":"active",
        "replicas":{
          "core_node4":{
            "core":"collection1_shard2_replica_n2",
            "leader":"true",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:46375/g_/p",
            "node_name":"127.0.0.1:46375_g_%2Fp",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node6":{
            "core":"collection1_shard2_replica_n5",
            "core_node_name":"core_node6",
            "base_url":"http://127.0.0.1:42559/g_/p",
            "node_name":"127.0.0.1:42559_g_%2Fp",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"}}

Stack Trace:
java.lang.AssertionError: Didn't see all replicas for shard shard1 in c8n_1x3_lf come up within 90000 ms! ClusterState: {
  "control_collection":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{"core_node2":{
            "core":"control_collection_shard1_replica_n1",
            "leader":"true",
            "core_node_name":"core_node2",
            "base_url":"http://127.0.0.1:44041/g_/p",
            "node_name":"127.0.0.1:44041_g_%2Fp",
            "state":"active",
            "collection":"control_collection",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"},
  "c8n_1x3_lf":{
    "pullReplicas":"0",
    "replicationFactor":"3",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"c8n_1x3_lf_shard1_replica_n1",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:44041/g_/p",
            "node_name":"127.0.0.1:44041_g_%2Fp",
            "state":"recovering",
            "collection":"c8n_1x3_lf",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node5":{
            "core":"c8n_1x3_lf_shard1_replica_n2",
            "leader":"true",
            "core_node_name":"core_node5",
            "base_url":"http://127.0.0.1:42559/g_/p",
            "node_name":"127.0.0.1:42559_g_%2Fp",
            "state":"active",
            "collection":"c8n_1x3_lf",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node6":{
            "core":"c8n_1x3_lf_shard1_replica_n4",
            "core_node_name":"core_node6",
            "base_url":"http://127.0.0.1:45159/g_/p",
            "node_name":"127.0.0.1:45159_g_%2Fp",
            "state":"recovering",
            "collection":"c8n_1x3_lf",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"3",
    "tlogReplicas":"0"},
  "collection1":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{
      "shard1":{
        "range":"80000000-ffffffff",
        "state":"active",
        "replicas":{"core_node3":{
            "core":"collection1_shard1_replica_n1",
            "leader":"true",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:45159/g_/p",
            "node_name":"127.0.0.1:45159_g_%2Fp",
            "state":"active",
            "collection":"collection1",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}},
      "shard2":{
        "range":"0-7fffffff",
        "state":"active",
        "replicas":{
          "core_node4":{
            "core":"collection1_shard2_replica_n2",
            "leader":"true",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:46375/g_/p",
            "node_name":"127.0.0.1:46375_g_%2Fp",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node6":{
            "core":"collection1_shard2_replica_n5",
            "core_node_name":"core_node6",
            "base_url":"http://127.0.0.1:42559/g_/p",
            "node_name":"127.0.0.1:42559_g_%2Fp",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"}}
	at __randomizedtesting.SeedInfo.seed([BEB7C845F8FAECE:83BF435EF173C336]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.ensureAllReplicasAreActive(AbstractFullDistribZkTestBase.java:2147)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:101)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:58)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:64)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:564)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1090)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1061)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:832)


FAILED:  org.apache.solr.cloud.LeaderFailureAfterFreshStartTest.test

Error Message:
java.lang.AssertionError: timeout waiting to see all nodes active

Stack Trace:
java.lang.AssertionError: timeout waiting to see all nodes active
	at __randomizedtesting.SeedInfo.seed([BEB7C845F8FAECE:83BF435EF173C336]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.solr.cloud.LeaderFailureAfterFreshStartTest.waitTillNodesActive(LeaderFailureAfterFreshStartTest.java:234)
	at org.apache.solr.cloud.LeaderFailureAfterFreshStartTest.restartNodes(LeaderFailureAfterFreshStartTest.java:174)
	at org.apache.solr.cloud.LeaderFailureAfterFreshStartTest.test(LeaderFailureAfterFreshStartTest.java:149)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:64)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/ja

[...truncated too long message...]

ests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseZGC" -Ptests.seed=BEB7C845F8FAECE -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=ISO-8859-1

  - org.apache.solr.cloud.LeaderVoteWaitTimeoutTest.testMostInSyncReplicasCanWinElection (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.LeaderVoteWaitTimeoutTest.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.LeaderVoteWaitTimeoutTest" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseZGC" -Ptests.seed=BEB7C845F8FAECE -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=ISO-8859-1

  - org.apache.solr.cloud.HttpPartitionTest.test (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.HttpPartitionTest.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.HttpPartitionTest" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseZGC" -Ptests.seed=BEB7C845F8FAECE -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=ISO-8859-1

  - org.apache.solr.cloud.SyncSliceTest.test (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.SyncSliceTest.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.SyncSliceTest" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseZGC" -Ptests.seed=BEB7C845F8FAECE -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=ISO-8859-1

  - org.apache.solr.cloud.UnloadDistributedZkTest.test (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.UnloadDistributedZkTest.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.UnloadDistributedZkTest" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseZGC" -Ptests.seed=BEB7C845F8FAECE -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=ISO-8859-1

  - org.apache.solr.cloud.MigrateRouteKeyTest.multipleShardMigrateTest (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.MigrateRouteKeyTest.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.MigrateRouteKeyTest" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseZGC" -Ptests.seed=BEB7C845F8FAECE -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=ISO-8859-1

  - org.apache.solr.cloud.DeleteReplicaTest.raceConditionOnDeleteAndRegisterReplica (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.DeleteReplicaTest.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.DeleteReplicaTest" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseZGC" -Ptests.seed=BEB7C845F8FAECE -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=ISO-8859-1

  - org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.RecoveryAfterSoftCommitTest.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.RecoveryAfterSoftCommitTest" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseZGC" -Ptests.seed=BEB7C845F8FAECE -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=ISO-8859-1

  - org.apache.solr.cloud.MoveReplicaTest.test (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.MoveReplicaTest.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.MoveReplicaTest" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseZGC" -Ptests.seed=BEB7C845F8FAECE -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=ISO-8859-1

  - org.apache.solr.cloud.TestLeaderElectionWithEmptyReplica.test (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.TestLeaderElectionWithEmptyReplica.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.TestLeaderElectionWithEmptyReplica" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseZGC" -Ptests.seed=BEB7C845F8FAECE -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=ISO-8859-1

  - org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseZGC" -Ptests.seed=BEB7C845F8FAECE -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=ISO-8859-1

  - org.apache.solr.cloud.RecoveryZkTest.test (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.RecoveryZkTest.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.RecoveryZkTest" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseZGC" -Ptests.seed=BEB7C845F8FAECE -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=ISO-8859-1

  - org.apache.solr.core.snapshots.TestSolrCloudSnapshots.testSnapshots (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.core.snapshots.TestSolrCloudSnapshots.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.core.snapshots.TestSolrCloudSnapshots" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseZGC" -Ptests.seed=BEB7C845F8FAECE -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=ISO-8859-1

  - org.apache.solr.cloud.TestCloudRecovery.corruptedLogTest (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.TestCloudRecovery.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.TestCloudRecovery" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseZGC" -Ptests.seed=BEB7C845F8FAECE -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=ISO-8859-1


Deprecated Gradle features were used in this build, making it incompatible with Gradle 7.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/6.6.1/userguide/command_line_interface.html#sec:command_line_warnings

BUILD SUCCESSFUL in 57m 27s
854 actionable tasks: 854 executed
Build step 'Invoke Gradle script' changed build result to SUCCESS
Archiving artifacts
java.lang.InterruptedException: no matches found within 10000
	at hudson.FilePath$ValidateAntFileMask.hasMatch(FilePath.java:2875)
	at hudson.FilePath$ValidateAntFileMask.invoke(FilePath.java:2754)
	at hudson.FilePath$ValidateAntFileMask.invoke(FilePath.java:2735)
	at hudson.FilePath.act(FilePath.java:1076)
	at hudson.FilePath.act(FilePath.java:1059)
	at hudson.FilePath.validateAntFileMask(FilePath.java:2733)
	at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:270)
	at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:80)
	at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
	at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:804)
	at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:753)
	at hudson.model.Build$BuildExecution.post2(Build.java:177)
	at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:698)
	at hudson.model.Run.execute(Run.java:1919)
	at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
	at hudson.model.ResourceController.execute(ResourceController.java:97)
	at hudson.model.Executor.run(Executor.java:428)
No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/*_pid*.log". Configuration error?
[Java] Sleeping for 5 seconds due to JENKINS-32191...
[Java] Parsing console log (workspace: '/home/jenkins/workspace/Lucene-Solr-master-Linux')
[Java] Successfully parsed console log
[Java] -> found 0 issues (skipped 0 duplicates)
[Java] Skipping post processing
[Java] No filter has been set, publishing all 0 issues
[Java] Using reference build 'Lucene-Solr-master-Linux #28363' to compute new, fixed, and outstanding issues
[Java] Issues delta (vs. reference build): outstanding: 0, new: 0, fixed: 0
[Java] No quality gates have been set - skipping
[Java] Health report is disabled - skipping
[Java] Created analysis result for 0 issues (found 0 new issues, fixed 0 issues)
[Java] Attaching ResultAction with ID 'java' to run 'Lucene-Solr-master-Linux #28364'.
[Checks API] No suitable checks publisher found.
Recording test results
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
[Checks API] No suitable checks publisher found.
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)

[JENKINS] Lucene-Solr-master-Linux (64bit/jdk-15) - Build # 28363 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Linux/28363/
Java: 64bit/jdk-15 -XX:-UseCompressedOops -XX:+UseSerialGC

52 tests failed.
FAILED:  org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest.test

Error Message:
java.lang.AssertionError: There are still nodes recovering - waited for 180SECONDS

Stack Trace:
java.lang.AssertionError: There are still nodes recovering - waited for 180SECONDS
	at __randomizedtesting.SeedInfo.seed([7251940245AE579A:FA05ABD8EB523A62]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:197)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:150)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:1063)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForThingsToLevelOut(AbstractFullDistribZkTestBase.java:1621)
	at org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest.test(ChaosMonkeySafeLeaderWithPullReplicasTest.java:198)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:64)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:564)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1090)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1061)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:832)


FAILED:  org.apache.solr.cloud.api.collections.TestHdfsCloudBackupRestore.test

Error Message:
java.lang.AssertionError: There are still nodes recovering - waited for 30SECONDS

Stack Trace:
java.lang.AssertionError: There are still nodes recovering - waited for 30SECONDS
	at __randomizedtesting.SeedInfo.seed([7251940245AE579A:FA05ABD8EB523A62]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:197)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:150)
	at org.apache.solr.cloud.api.collections.AbstractCloudBackupRestoreTestCase.testBackupAndRestore(AbstractCloudBackupRestoreTestCase.java:353)
	at org.apache.solr.cloud.api.collections.AbstractCloudBackupRestoreTestCase.test(AbstractCloudBackupRestoreTestCase.java:155)
	at org.apache.solr.cloud.api.collections.TestHdfsCloudBackupRestore.test(TestHdfsCloudBackupRestore.java:225)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:64)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:564)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:832)


FAILED:  org.apache.solr.cloud.api.collections.TestLocalFSCloudBackupRestore.test

Error Message:
java.lang.AssertionError: There are still nodes recovering - waited for 30SECONDS

Stack Trace:
java.lang.AssertionError: There are still nodes recovering - waited for 30SECONDS
	at __randomizedtesting.SeedInfo.seed([7251940245AE579A:FA05ABD8EB523A62]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:197)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:150)
	at org.apache.solr.cloud.api.collections.AbstractCloudBackupRestoreTestCase.testBackupAndRestore(AbstractCloudBackupRestoreTestCase.java:353)
	at org.apache.solr.cloud.api.collections.AbstractCloudBackupRestoreTestCase.test(AbstractCloudBackupRestoreTestCase.java:155)
	at org.apache.solr.cloud.api.collections.TestLocalFSCloudBackupRestore.test(TestLocalFSCloudBackupRestore.java:93)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:64)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:564)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:832)


FAILED:  org.apache.solr.cloud.ReplaceNodeTest.test

Error Message:
java.lang.AssertionError: Expecting no cores but found some: {replacenodetest_coll_shard4_replica_n11={name=replacenodetest_coll_shard4_replica_n11, instanceDir=/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/tmp/tests-tmp/solr.cloud.ReplaceNodeTest_7251940245AE579A-001/tempDir-001/node6/replacenodetest_coll_shard4_replica_n11, dataDir=/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/tmp/tests-tmp/solr.cloud.ReplaceNodeTest_7251940245AE579A-001/tempDir-001/node6/replacenodetest_coll_shard4_replica_n11/data/, config=solrconfig.xml, schema=schema.xml, startTime=Sun Oct 18 13:16:55 EGST 2020, uptime=17277, lastPublished=active, configVersion=0, cloud={collection=replacenodetest_coll, shard=shard4, replica=core_node12, replicaType=NRT}, index={numDocs=0, maxDoc=0, deletedDocs=0, indexHeapUsageBytes=0, version=3, segmentCount=0, current=false, hasDeletions=false, directory=org.apache.lucene.store.MockDirectoryWrapper:MockDirectoryWrapper(ByteBuffersDirectory@4736b7a5 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@3d51ad30), segmentsFile=segments_1, segmentsFileSizeInBytes=-1, userData={}, sizeInBytes=135, size=135 bytes}}} expected:<0> but was:<1>

Stack Trace:
java.lang.AssertionError: Expecting no cores but found some: {replacenodetest_coll_shard4_replica_n11={name=replacenodetest_coll_shard4_replica_n11, instanceDir=/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/tmp/tests-tmp/solr.cloud.ReplaceNodeTest_7251940245AE579A-001/tempDir-001/node6/replacenodetest_coll_shard4_replica_n11, dataDir=/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/tmp/tests-tmp/solr.cloud.ReplaceNodeTest_7251940245AE579A-001/tempDir-001/node6/replacenodetest_coll_shard4_replica_n11/data/, config=solrconfig.xml, schema=schema.xml, startTime=Sun Oct 18 13:16:55 EGST 2020, uptime=17277, lastPublished=active, configVersion=0, cloud={collection=replacenodetest_coll, shard=shard4, replica=core_node12, replicaType=NRT}, index={numDocs=0, maxDoc=0, deletedDocs=0, indexHeapUsageBytes=0, version=3, segmentCount=0, current=false, hasDeletions=false, directory=org.apache.lucene.store.MockDirectoryWrapper:MockDirectoryWrapper(ByteBuffersDirectory@4736b7a5 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@3d51ad30), segmentsFile=segments_1, segmentsFileSizeInBytes=-1, userData={}, sizeInBytes=135, size=135 bytes}}} expected:<0> but was:<1>
	at __randomizedtesting.SeedInfo.seed([7251940245AE579A:FA05ABD8EB523A62]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.junit.Assert.failNotEquals(Assert.java:835)
	at org.junit.Assert.assertEquals(Assert.java:647)
	at org.apache.solr.cloud.ReplaceNodeTest.test(ReplaceNodeTest.java:148)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:64)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:564)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:832)


FAILED:  org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest.test

Error Message:
java.lang.AssertionError: There are still nodes recovering - waited for 600SECONDS

Stack Trace:
java.lang.AssertionError: There are still nodes recovering - waited for 600SECONDS
	at __randomizedtesting.SeedInfo.seed([7251940245AE579A:FA05ABD8EB523A62]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:197)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:150)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:1063)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForThingsToLevelOut(AbstractFullDistribZkTestBase.java:1621)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForThingsToLevelOut(AbstractFullDistribZkTestBase.java:1613)
	at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest.test(ChaosMonkeyNothingIsSafeWithPullReplicasTest.java:253)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:64)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:564)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1090)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1061)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:832)


FAILED:  org.apache.solr.cloud.MoveReplicaTest.test

Error Message:
java.lang.AssertionError

Stack Trace:
java.lang.AssertionError
	at __randomizedtesting.SeedInfo.seed([7251940245AE579A:FA05ABD8EB523A62]:0)
	at org.junit.Assert.fail(Assert.java:87)
	at org.junit.Assert.assertTrue(Assert.java:42)
	at org.junit.Assert.assertTrue(Assert.java:53)
	at org.apache.solr.cloud.MoveReplicaTest.test(MoveReplicaTest.java:150)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:64)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:564)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:832)


FAILED:  org.apache.solr.cloud.BasicDistributedZk2Test.test

Error Message:
java.lang.AssertionError: There are still nodes recovering - waited for 330SECONDS

Stack Trace:
java.lang.AssertionError: There are still nodes recovering - waited for 330SECONDS
	at __randomizedtesting.SeedInfo.seed([7251940245AE579A:FA05ABD8EB523A62]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:197)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:144)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:139)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:1051)
	at org.apache.solr.cloud.BasicDistributedZk2Test.brindDownShardIndexSomeDocsAndRecover(BasicDistributedZk2Test.java:362)
	at org.apache.solr.cloud.BasicDistributedZk2Test.test(BasicDistributedZk2Test.java:103)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:64)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:564)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1090)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1061)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:832)


FAILED:  org.apache.solr.cloud.DeleteReplicaTest.raceConditionOnDeleteAndRegisterReplica

Error Message:
java.lang.RuntimeException: Failed while waiting for active collection
Timeout waiting to see state for collection=raceDeleteReplicaCollection :DocCollection(raceDeleteReplicaCollection/15)={
  "pullReplicas":"0",
  "replicationFactor":"2",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node4":{
          "core":"raceDeleteReplicaCollection_shard1_replica_n2",
          "core_node_name":"core_node4",
          "base_url":"http://127.0.0.1:40453/solr",
          "node_name":"127.0.0.1:40453_solr",
          "state":"recovering",
          "collection":"raceDeleteReplicaCollection",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node6":{
          "core":"raceDeleteReplicaCollection_shard1_replica_n5",
          "leader":"true",
          "core_node_name":"core_node6",
          "base_url":"http://127.0.0.1:44487/solr",
          "node_name":"127.0.0.1:44487_solr",
          "state":"active",
          "collection":"raceDeleteReplicaCollection",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"2",
  "tlogReplicas":"0"}
Live Nodes: [127.0.0.1:40453_solr, 127.0.0.1:43421_solr, 127.0.0.1:44295_solr, 127.0.0.1:44487_solr]
Last available state: DocCollection(raceDeleteReplicaCollection/15)={
  "pullReplicas":"0",
  "replicationFactor":"2",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node4":{
          "core":"raceDeleteReplicaCollection_shard1_replica_n2",
          "core_node_name":"core_node4",
          "base_url":"http://127.0.0.1:40453/solr",
          "node_name":"127.0.0.1:40453_solr",
          "state":"recovering",
          "collection":"raceDeleteReplicaCollection",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node6":{
          "core":"raceDeleteReplicaCollection_shard1_replica_n5",
          "leader":"true",
          "core_node_name":"core_node6",
          "base_url":"http://127.0.0.1:44487/solr",
          "node_name":"127.0.0.1:44487_solr",
          "state":"active",
          "collection":"raceDeleteReplicaCollection",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"2",
  "tlogReplicas":"0"}

Stack Trace:
java.lang.RuntimeException: Failed while waiting for active collection
Timeout waiting to see state for collection=raceDeleteReplicaCollection :DocCollection(raceDeleteReplicaCollection/15)={
  "pullReplicas":"0",
  "replicationFactor":"2",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node4":{
          "core":"raceDeleteReplicaCollection_shard1_replica_n2",
          "core_node_name":"core_node4",
          "base_url":"http://127.0.0.1:40453/solr",
          "node_name":"127.0.0.1:40453_solr",
          "state":"recovering",
          "collection":"raceDeleteReplicaCollection",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node6":{
          "core":"raceDeleteReplicaCollection_shard1_replica_n5",
          "leader":"true",
          "core_node_name":"core_node6",
          "base_url":"http://127.0.0.1:44487/solr",
          "node_name":"127.0.0.1:44487_solr",
          "state":"active",
          "collection":"raceDeleteReplicaCollection",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"2",
  "tlogReplicas":"0"}
Live Nodes: [127.0.0.1:40453_solr, 127.0.0.1:43421_solr, 127.0.0.1:44295_solr, 127.0.0.1:44487_solr]
Last available state: DocCollection(raceDeleteReplicaCollection/15)={
  "pullReplicas":"0",
  "replicationFactor":"2",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node4":{
          "core":"raceDeleteReplicaCollection_shard1_replica_n2",
          "core_node_name":"core_node4",
          "base_url":"http://127.0.0.1:40453/solr",
          "node_name":"127.0.0.1:40453_solr",
          "state":"recovering",
          "collection":"raceDeleteReplicaCollection",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node6":{
          "core":"raceDeleteReplicaCollection_shard1_replica_n5",
          "leader":"true",
          "core_node_name":"core_node6",
          "base_url":"http://127.0.0.1:44487/solr",
          "node_name":"127.0.0.1:44487_solr",
          "state":"active",
          "collection":"raceDeleteReplicaCollection",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"2",
  "tlogReplicas":"0"}
	at __randomizedtesting.SeedInfo.seed([7251940245AE579A:1847F5D22D5C1D50]:0)
	at org.apache.solr.cloud.MiniSolrCloudCluster.waitForActiveCollection(MiniSolrCloudCluster.java:774)
	at org.apache.solr.cloud.MiniSolrCloudCluster.waitForActiveCollection(MiniSolrCloudCluster.java:780)
	at org.apache.solr.cloud.DeleteReplicaTest.raceConditionOnDeleteAndRegisterReplica(DeleteReplicaTest.java:385)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:64)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:564)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:832)


FAILED:  org.apache.solr.cloud.HttpPartitionTest.test

Error Message:
java.lang.AssertionError: Didn't see all replicas for shard shard1 in collDoRecoveryOnRestart come up within 130000 ms! ClusterState: {
  "control_collection":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{"core_node2":{
            "core":"control_collection_shard1_replica_n1",
            "leader":"true",
            "core_node_name":"core_node2",
            "base_url":"http://127.0.0.1:37857/tp_qj/l",
            "node_name":"127.0.0.1:37857_tp_qj%2Fl",
            "state":"active",
            "collection":"control_collection",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"},
  "collDoRecoveryOnRestart":{
    "pullReplicas":"0",
    "replicationFactor":"2",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"collDoRecoveryOnRestart_shard1_replica_n1",
            "leader":"true",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:35997/tp_qj/l",
            "node_name":"127.0.0.1:35997_tp_qj%2Fl",
            "state":"active",
            "collection":"collDoRecoveryOnRestart",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node4":{
            "core":"collDoRecoveryOnRestart_shard1_replica_n2",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:37857/tp_qj/l",
            "node_name":"127.0.0.1:37857_tp_qj%2Fl",
            "state":"recovering",
            "collection":"collDoRecoveryOnRestart",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"2",
    "tlogReplicas":"0"},
  "collection1":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{
      "shard1":{
        "range":"80000000-ffffffff",
        "state":"active",
        "replicas":{"core_node4":{
            "core":"collection1_shard1_replica_n2",
            "leader":"true",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:44037/tp_qj/l",
            "node_name":"127.0.0.1:44037_tp_qj%2Fl",
            "state":"active",
            "collection":"collection1",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}},
      "shard2":{
        "range":"0-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"collection1_shard2_replica_n1",
            "leader":"true",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:43179/tp_qj/l",
            "node_name":"127.0.0.1:43179_tp_qj%2Fl",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node6":{
            "core":"collection1_shard2_replica_n5",
            "core_node_name":"core_node6",
            "base_url":"http://127.0.0.1:35997/tp_qj/l",
            "node_name":"127.0.0.1:35997_tp_qj%2Fl",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"}}

Stack Trace:
java.lang.AssertionError: Didn't see all replicas for shard shard1 in collDoRecoveryOnRestart come up within 130000 ms! ClusterState: {
  "control_collection":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{"core_node2":{
            "core":"control_collection_shard1_replica_n1",
            "leader":"true",
            "core_node_name":"core_node2",
            "base_url":"http://127.0.0.1:37857/tp_qj/l",
            "node_name":"127.0.0.1:37857_tp_qj%2Fl",
            "state":"active",
            "collection":"control_collection",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"},
  "collDoRecoveryOnRestart":{
    "pullReplicas":"0",
    "replicationFactor":"2",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"collDoRecoveryOnRestart_shard1_replica_n1",
            "leader":"true",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:35997/tp_qj/l",
            "node_name":"127.0.0.1:35997_tp_qj%2Fl",
            "state":"active",
            "collection":"collDoRecoveryOnRestart",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node4":{
            "core":"collDoRecoveryOnRestart_shard1_replica_n2",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:37857/tp_qj/l",
            "node_name":"127.0.0.1:37857_tp_qj%2Fl",
            "state":"recovering",
            "collection":"collDoRecoveryOnRestart",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"2",
    "tlogReplicas":"0"},
  "collection1":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{
      "shard1":{
        "range":"80000000-ffffffff",
        "state":"active",
        "replicas":{"core_node4":{
            "core":"collection1_shard1_replica_n2",
            "leader":"true",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:44037/tp_qj/l",
            "node_name":"127.0.0.1:44037_tp_qj%2Fl",
            "state":"active",
            "collection":"collection1",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}},
      "shard2":{
        "range":"0-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"collection1_shard2_replica_n1",
            "leader":"true",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:43179/tp_qj/l",
            "node_name":"127.0.0.1:43179_tp_qj%2Fl",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node6":{
            "core":"collection1_shard2_replica_n5",
            "core_node_name":"core_node6",
            "base_url":"http://127.0.0.1:35997/tp_qj/l",
            "node_name":"127.0.0.1:35997_tp_qj%2Fl",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"}}
	at __randomizedtesting.SeedInfo.seed([7251940245AE579A:FA05ABD8EB523A62]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.ensureAllReplicasAreActive(AbstractFullDistribZkTestBase.java:2147)
	at org.apache.solr.cloud.HttpPartitionTest.testDoRecoveryOnRestart(HttpPartitionTest.java:200)
	at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:132)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:64)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:564)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1090)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1061)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:832)


FAILED:  org.apache.solr.cloud.HttpPartitionWithTlogReplicasTest.test

Error Message:
java.lang.AssertionError: Didn't see all replicas for shard shard1 in collDoRecoveryOnRestart come up within 130000 ms! ClusterState: {
  "control_collection":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{"core_node2":{
            "core":"control_collection_shard1_replica_n1",
            "leader":"true",
            "core_node_name":"core_node2",
            "base_url":"http://127.0.0.1:37749/agi",
            "node_name":"127.0.0.1:37749_agi",
            "state":"active",
            "collection":"control_collection",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"},
  "collDoRecoveryOnRestart":{
    "pullReplicas":"0",
    "replicationFactor":"0",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"collDoRecoveryOnRestart_shard1_replica_t1",
            "leader":"true",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:37749/agi",
            "node_name":"127.0.0.1:37749_agi",
            "state":"active",
            "collection":"collDoRecoveryOnRestart",
            "shard":"shard1",
            "type":"TLOG",
            "force_set_state":"false"},
          "core_node4":{
            "core":"collDoRecoveryOnRestart_shard1_replica_t2",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:45009/agi",
            "node_name":"127.0.0.1:45009_agi",
            "state":"recovering",
            "collection":"collDoRecoveryOnRestart",
            "shard":"shard1",
            "type":"TLOG",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"0",
    "tlogReplicas":"2"},
  "collection1":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{
      "shard1":{
        "range":"80000000-ffffffff",
        "state":"active",
        "replicas":{"core_node2":{
            "core":"collection1_shard1_replica_t1",
            "leader":"true",
            "core_node_name":"core_node2",
            "base_url":"http://127.0.0.1:46151/agi",
            "node_name":"127.0.0.1:46151_agi",
            "state":"active",
            "collection":"collection1",
            "shard":"shard1",
            "type":"TLOG",
            "force_set_state":"false"}}},
      "shard2":{
        "range":"0-7fffffff",
        "state":"active",
        "replicas":{
          "core_node4":{
            "core":"collection1_shard2_replica_t3",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:45009/agi",
            "node_name":"127.0.0.1:45009_agi",
            "state":"recovering",
            "collection":"collection1",
            "shard":"shard2",
            "type":"TLOG",
            "force_set_state":"false"},
          "core_node6":{
            "core":"collection1_shard2_replica_t5",
            "leader":"true",
            "core_node_name":"core_node6",
            "base_url":"http://127.0.0.1:38591/agi",
            "node_name":"127.0.0.1:38591_agi",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"TLOG",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"}}

Stack Trace:
java.lang.AssertionError: Didn't see all replicas for shard shard1 in collDoRecoveryOnRestart come up within 130000 ms! ClusterState: {
  "control_collection":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{"core_node2":{
            "core":"control_collection_shard1_replica_n1",
            "leader":"true",
            "core_node_name":"core_node2",
            "base_url":"http://127.0.0.1:37749/agi",
            "node_name":"127.0.0.1:37749_agi",
            "state":"active",
            "collection":"control_collection",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"},
  "collDoRecoveryOnRestart":{
    "pullReplicas":"0",
    "replicationFactor":"0",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"collDoRecoveryOnRestart_shard1_replica_t1",
            "leader":"true",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:37749/agi",
            "node_name":"127.0.0.1:37749_agi",
            "state":"active",
            "collection":"collDoRecoveryOnRestart",
            "shard":"shard1",
            "type":"TLOG",
            "force_set_state":"false"},
          "core_node4":{
            "core":"collDoRecoveryOnRestart_shard1_replica_t2",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:45009/agi",
            "node_name":"127.0.0.1:45009_agi",
            "state":"recovering",
            "collection":"collDoRecoveryOnRestart",
            "shard":"shard1",
            "type":"TLOG",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"0",
    "tlogReplicas":"2"},
  "collection1":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{
      "shard1":{
        "range":"80000000-ffffffff",
        "state":"active",
        "replicas":{"core_node2":{
            "core":"collection1_shard1_replica_t1",
            "leader":"true",
            "core_node_name":"core_node2",
            "base_url":"http://127.0.0.1:46151/agi",
            "node_name":"127.0.0.1:46151_agi",
            "state":"active",
            "collection":"collection1",
            "shard":"shard1",
            "type":"TLOG",
            "force_set_state":"false"}}},
      "shard2":{
        "range":"0-7fffffff",
        "state":"active",
        "replicas":{
          "core_node4":{
            "core":"collection1_shard2_replica_t3",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:45009/agi",
            "node_name":"127.0.0.1:45009_agi",
            "state":"recovering",
            "collection":"collection1",
            "shard":"shard2",
            "type":"TLOG",
            "force_set_state":"false"},
          "core_node6":{
            "core":"collection1_shard2_replica_t5",
            "leader":"true",
            "core_node_name":"core_node6",
            "base_url":"http://127.0.0.1:38591/agi",
            "node_name":"127.0.0.1:38591_agi",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"TLOG",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"}}
	at __randomizedtesting.SeedInfo.seed([7251940245AE579A:FA05ABD8EB523A62]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.ensureAllReplicasAreActive(AbstractFullDistribZkTestBase.java:2147)
	at org.apache.solr.cloud.HttpPartitionTest.testDoRecoveryOnRestart(HttpPartitionTest.java:200)
	at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:132)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:64)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:564)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1090)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1061)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:832)


FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test

Error Message:
java.lang.AssertionError: Didn't see all replicas for shard shard1 in c8n_1x3_lf come up within 90000 ms! ClusterState: {
  "control_collection":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{"core_node2":{
            "core":"control_collection_shard1_replica_n1",
            "leader":"true",
            "core_node_name":"core_node2",
            "base_url":"http://127.0.0.1:39579",
            "node_name":"127.0.0.1:39579_",
            "state":"active",
            "collection":"control_collection",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"},
  "c8n_1x3_lf":{
    "pullReplicas":"0",
    "replicationFactor":"3",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"c8n_1x3_lf_shard1_replica_n1",
            "leader":"true",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:44881",
            "node_name":"127.0.0.1:44881_",
            "state":"active",
            "collection":"c8n_1x3_lf",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node5":{
            "core":"c8n_1x3_lf_shard1_replica_n2",
            "core_node_name":"core_node5",
            "base_url":"http://127.0.0.1:39733",
            "node_name":"127.0.0.1:39733_",
            "state":"recovering",
            "collection":"c8n_1x3_lf",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node6":{
            "core":"c8n_1x3_lf_shard1_replica_n4",
            "core_node_name":"core_node6",
            "base_url":"http://127.0.0.1:39579",
            "node_name":"127.0.0.1:39579_",
            "state":"recovering",
            "collection":"c8n_1x3_lf",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"3",
    "tlogReplicas":"0"},
  "collection1":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{
      "shard1":{
        "range":"80000000-ffffffff",
        "state":"active",
        "replicas":{"core_node4":{
            "core":"collection1_shard1_replica_n3",
            "leader":"true",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:39733",
            "node_name":"127.0.0.1:39733_",
            "state":"active",
            "collection":"collection1",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}},
      "shard2":{
        "range":"0-7fffffff",
        "state":"active",
        "replicas":{
          "core_node2":{
            "core":"collection1_shard2_replica_n1",
            "leader":"true",
            "core_node_name":"core_node2",
            "base_url":"http://127.0.0.1:44881",
            "node_name":"127.0.0.1:44881_",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node6":{
            "core":"collection1_shard2_replica_n5",
            "core_node_name":"core_node6",
            "base_url":"http://127.0.0.1:41613",
            "node_name":"127.0.0.1:41613_",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"}}

Stack Trace:
java.lang.AssertionError: Didn't see all replicas for shard shard1 in c8n_1x3_lf come up within 90000 ms! ClusterState: {
  "control_collection":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{"core_node2":{
            "core":"control_collection_shard1_replica_n1",
            "leader":"true",
            "core_node_name":"core_node2",
            "base_url":"http://127.0.0.1:39579",
            "node_name":"127.0.0.1:39579_",
            "state":"active",
            "collection":"control_collection",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"},
  "c8n_1x3_lf":{
    "pullReplicas":"0",
    "replicationFactor":"3",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"c8n_1x3_lf_shard1_replica_n1",
            "leader":"true",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:44881",
            "node_name":"127.0.0.1:44881_",
            "state":"active",
            "collection":"c8n_1x3_lf",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node5":{
            "core":"c8n_1x3_lf_shard1_replica_n2",
            "core_node_name":"core_node5",
            "base_url":"http://127.0.0.1:39733",
            "node_name":"127.0.0.1:39733_",
            "state":"recovering",
            "collection":"c8n_1x3_lf",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node6":{
            "core":"c8n_1x3_lf_shard1_replica_n4",
            "core_node_name":"core_node6",
            "base_url":"http://127.0.0.1:39579",
            "node_name":"127.0.0.1:39579_",
            "state":"recovering",
            "collection":"c8n_1x3_lf",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"3",
    "tlogReplicas":"0"},
  "collection1":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{
      "shard1":{
        "range":"80000000-ffffffff",
        "state":"active",
        "replicas":{"core_node4":{
            "core":"collection1_shard1_replica_n3",
            "leader":"true",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:39733",
            "node_name":"127.0.0.1:39733_",
            "state":"active",
            "collection":"collection1",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}},
      "shard2":{
        "range":"0-7fffffff",
        "state":"active",
        "replicas":{
          "core_node2":{
            "core":"collection1_shard2_replica_n1",
            "leader":"true",
            "core_node_name":"core_node2",
            "base_url":"http://127.0.0.1:44881",
            "node_name":"127.0.0.1:44881_",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node6":{
            "core":"collection1_shard2_replica_n5",
            "core_node_name":"core_node6",
            "base_url":"http://127.0.0.1:41613",
            "node_name":"127.0.0.1:41613_",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"}}
	at __randomizedtesting.SeedInfo.seed([7251940245AE579A:FA05ABD8EB523A62]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.ensureAllReplicasAreActive(AbstractFullDistribZkTestBase.java:2147)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:101)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:58)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:64)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:564)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1090)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1061)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:832)


FAILED:  org.apache.solr.cloud.LeaderFailureAfterFreshStartTest.test

Error Message:
java.lang.AssertionError: timeout waiting to see all nodes active

Stack Trace:
java.lang.AssertionError: timeout waiting to see all nodes active
	at __randomizedtesting.SeedInfo.seed([7251940245AE579A:FA05ABD8EB523A62]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.solr.cloud.LeaderFailureAfterFreshStartTest.waitTillNodesActive(LeaderFailureAfterFreshStartTest.java:234)
	at org.apache.solr.cloud.LeaderFailureAfterFreshStartTest.restartNodes(LeaderFailureAfterFreshStartTest.java:174)
	at org.apache.solr.cloud.LeaderFailureAfterFreshStartTest.test(LeaderFailureAfterFreshStartTest.java:149)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:64)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:564)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1090)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1061)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:832)


FAILED:  org.apache.solr.cloud.LeaderVoteWaitTimeoutTest.testMostInSyncReplicasCanWinElection

Error Message:
java.lang.AssertionError: Timeout waiting for 1x3 collection
Timeout waiting to see state for collection=collection1 :DocCollection(collection1/26)={
  "pullReplicas":"0",
  "replicationFactor":"3",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node2":{
          "core":"collection1_shard1_replica_n1",
          "core_node_name":"core_node2",
          "base_url":"https://127.0.0.1:44927/solr",
          "node_name":"127.0.0.1:44927_solr",
          "state":"recovering",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node4":{
          "core":"collection1_shard1_replica_n3",


[...truncated too long message...]

eSerialGC" -Ptests.seed=7251940245AE579A -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=ISO-8859-1

  - org.apache.solr.cloud.HttpPartitionTest.test (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.HttpPartitionTest.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.HttpPartitionTest" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseSerialGC" -Ptests.seed=7251940245AE579A -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=ISO-8859-1

  - org.apache.solr.cloud.SyncSliceTest.test (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.SyncSliceTest.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.SyncSliceTest" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseSerialGC" -Ptests.seed=7251940245AE579A -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=ISO-8859-1

  - org.apache.solr.cloud.MigrateRouteKeyTest.multipleShardMigrateTest (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.MigrateRouteKeyTest.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.MigrateRouteKeyTest" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseSerialGC" -Ptests.seed=7251940245AE579A -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=ISO-8859-1

  - org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.RecoveryAfterSoftCommitTest.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.RecoveryAfterSoftCommitTest" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseSerialGC" -Ptests.seed=7251940245AE579A -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=ISO-8859-1

  - org.apache.solr.cloud.ReplaceNodeTest.test (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.ReplaceNodeTest.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.ReplaceNodeTest" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseSerialGC" -Ptests.seed=7251940245AE579A -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=ISO-8859-1

  - org.apache.solr.cloud.TestLeaderElectionWithEmptyReplica.test (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.TestLeaderElectionWithEmptyReplica.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.TestLeaderElectionWithEmptyReplica" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseSerialGC" -Ptests.seed=7251940245AE579A -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=ISO-8859-1

  - org.apache.solr.cloud.DeleteReplicaTest.raceConditionOnDeleteAndRegisterReplica (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.DeleteReplicaTest.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.DeleteReplicaTest" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseSerialGC" -Ptests.seed=7251940245AE579A -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=ISO-8859-1

  - org.apache.solr.cloud.LeaderFailureAfterFreshStartTest.test (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.LeaderFailureAfterFreshStartTest.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.LeaderFailureAfterFreshStartTest" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseSerialGC" -Ptests.seed=7251940245AE579A -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=ISO-8859-1

  - org.apache.solr.cloud.LeaderVoteWaitTimeoutTest.testMostInSyncReplicasCanWinElection (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.LeaderVoteWaitTimeoutTest.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.LeaderVoteWaitTimeoutTest" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseSerialGC" -Ptests.seed=7251940245AE579A -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=ISO-8859-1

  - org.apache.solr.cloud.MoveReplicaTest.test (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.MoveReplicaTest.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.MoveReplicaTest" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseSerialGC" -Ptests.seed=7251940245AE579A -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=ISO-8859-1

  - org.apache.solr.core.snapshots.TestSolrCloudSnapshots.testSnapshots (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.core.snapshots.TestSolrCloudSnapshots.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.core.snapshots.TestSolrCloudSnapshots" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseSerialGC" -Ptests.seed=7251940245AE579A -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=ISO-8859-1

  - org.apache.solr.cloud.TestCloudRecovery.corruptedLogTest (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.TestCloudRecovery.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.TestCloudRecovery" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseSerialGC" -Ptests.seed=7251940245AE579A -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=ISO-8859-1

  - org.apache.solr.cloud.RecoveryZkTest.test (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.RecoveryZkTest.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.RecoveryZkTest" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseSerialGC" -Ptests.seed=7251940245AE579A -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=ISO-8859-1


Deprecated Gradle features were used in this build, making it incompatible with Gradle 7.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/6.6.1/userguide/command_line_interface.html#sec:command_line_warnings

BUILD SUCCESSFUL in 57m 44s
854 actionable tasks: 854 executed
Build step 'Invoke Gradle script' changed build result to SUCCESS
Archiving artifacts
java.lang.InterruptedException: no matches found within 10000
	at hudson.FilePath$ValidateAntFileMask.hasMatch(FilePath.java:2875)
	at hudson.FilePath$ValidateAntFileMask.invoke(FilePath.java:2754)
	at hudson.FilePath$ValidateAntFileMask.invoke(FilePath.java:2735)
	at hudson.FilePath.act(FilePath.java:1076)
	at hudson.FilePath.act(FilePath.java:1059)
	at hudson.FilePath.validateAntFileMask(FilePath.java:2733)
	at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:270)
	at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:80)
	at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
	at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:804)
	at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:753)
	at hudson.model.Build$BuildExecution.post2(Build.java:177)
	at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:698)
	at hudson.model.Run.execute(Run.java:1919)
	at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
	at hudson.model.ResourceController.execute(ResourceController.java:97)
	at hudson.model.Executor.run(Executor.java:428)
No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/*_pid*.log". Configuration error?
[Java] Sleeping for 5 seconds due to JENKINS-32191...
[Java] Parsing console log (workspace: '/home/jenkins/workspace/Lucene-Solr-master-Linux')
[Java] Successfully parsed console log
[Java] -> found 0 issues (skipped 0 duplicates)
[Java] Skipping post processing
[Java] No filter has been set, publishing all 0 issues
[Java] Using reference build 'Lucene-Solr-master-Linux #28362' to compute new, fixed, and outstanding issues
[Java] Issues delta (vs. reference build): outstanding: 0, new: 0, fixed: 0
[Java] No quality gates have been set - skipping
[Java] Health report is disabled - skipping
[Java] Created analysis result for 0 issues (found 0 new issues, fixed 0 issues)
[Java] Attaching ResultAction with ID 'java' to run 'Lucene-Solr-master-Linux #28363'.
[Checks API] No suitable checks publisher found.
Recording test results
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
[Checks API] No suitable checks publisher found.
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)

[JENKINS] Lucene-Solr-master-Linux (64bit/jdk-13.0.2) - Build # 28362 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Linux/28362/
Java: 64bit/jdk-13.0.2 -XX:-UseCompressedOops -XX:+UseSerialGC

53 tests failed.
FAILED:  org.apache.lucene.search.spans.TestSpanExplanations.testExplainWithoutScoring

Error Message:
java.lang.AssertionError: The top-reader used to create Weight is not the same as the current reader's top-reader (org.apache.lucene.index.CompositeReaderContext@3b1c215d

Stack Trace:
java.lang.AssertionError: The top-reader used to create Weight is not the same as the current reader's top-reader (org.apache.lucene.index.CompositeReaderContext@3b1c215d
	at __randomizedtesting.SeedInfo.seed([AF6A214F1A34E7EE:3C7E00E17CBFDA59]:0)
	at org.apache.lucene.search.spans.SpanTermQuery$SpanTermWeight.getSpans(SpanTermQuery.java:121)
	at org.apache.lucene.search.spans.SpanNearQuery$SpanNearWeight.getSpans(SpanNearQuery.java:220)
	at org.apache.lucene.search.spans.SpanWeight.scorer(SpanWeight.java:136)
	at org.apache.lucene.search.spans.SpanWeight.explain(SpanWeight.java:156)
	at org.apache.lucene.search.spans.TestSpanExplanations.testExplainWithoutScoring(TestSpanExplanations.java:197)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:567)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:830)


FAILED:  org.apache.lucene.search.spans.TestSpanExplanationsOfNonMatches.testExplainWithoutScoring

Error Message:
java.lang.AssertionError: The top-reader used to create Weight is not the same as the current reader's top-reader (org.apache.lucene.index.CompositeReaderContext@16b70fab

Stack Trace:
java.lang.AssertionError: The top-reader used to create Weight is not the same as the current reader's top-reader (org.apache.lucene.index.CompositeReaderContext@16b70fab
	at __randomizedtesting.SeedInfo.seed([AF6A214F1A34E7EE:3C7E00E17CBFDA59]:0)
	at org.apache.lucene.search.spans.SpanTermQuery$SpanTermWeight.getSpans(SpanTermQuery.java:121)
	at org.apache.lucene.search.spans.SpanNearQuery$SpanNearWeight.getSpans(SpanNearQuery.java:220)
	at org.apache.lucene.search.spans.SpanWeight.scorer(SpanWeight.java:136)
	at org.apache.lucene.search.spans.SpanWeight.explain(SpanWeight.java:156)
	at org.apache.lucene.search.spans.TestSpanExplanations.testExplainWithoutScoring(TestSpanExplanations.java:197)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:567)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:830)


FAILED:  org.apache.solr.cloud.api.collections.TestHdfsCloudBackupRestore.test

Error Message:
java.lang.AssertionError: There are still nodes recovering - waited for 30SECONDS

Stack Trace:
java.lang.AssertionError: There are still nodes recovering - waited for 30SECONDS
	at __randomizedtesting.SeedInfo.seed([AF6A214F1A34E7EE:273E1E95B4C88A16]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:197)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:150)
	at org.apache.solr.cloud.api.collections.AbstractCloudBackupRestoreTestCase.testBackupAndRestore(AbstractCloudBackupRestoreTestCase.java:353)
	at org.apache.solr.cloud.api.collections.AbstractCloudBackupRestoreTestCase.test(AbstractCloudBackupRestoreTestCase.java:155)
	at org.apache.solr.cloud.api.collections.TestHdfsCloudBackupRestore.test(TestHdfsCloudBackupRestore.java:225)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:567)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:830)


FAILED:  org.apache.solr.cloud.api.collections.TestLocalFSCloudBackupRestore.test

Error Message:
java.lang.AssertionError: There are still nodes recovering - waited for 30SECONDS

Stack Trace:
java.lang.AssertionError: There are still nodes recovering - waited for 30SECONDS
	at __randomizedtesting.SeedInfo.seed([AF6A214F1A34E7EE:273E1E95B4C88A16]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:197)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:150)
	at org.apache.solr.cloud.api.collections.AbstractCloudBackupRestoreTestCase.testBackupAndRestore(AbstractCloudBackupRestoreTestCase.java:353)
	at org.apache.solr.cloud.api.collections.AbstractCloudBackupRestoreTestCase.test(AbstractCloudBackupRestoreTestCase.java:155)
	at org.apache.solr.cloud.api.collections.TestLocalFSCloudBackupRestore.test(TestLocalFSCloudBackupRestore.java:93)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:567)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:830)


FAILED:  org.apache.solr.cloud.ReplaceNodeTest.test

Error Message:
java.lang.AssertionError: Expecting no cores but found some: {replacenodetest_coll_shard5_replica_n11={name=replacenodetest_coll_shard5_replica_n11, instanceDir=/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/tmp/tests-tmp/solr.cloud.ReplaceNodeTest_AF6A214F1A34E7EE-001/tempDir-001/node4/replacenodetest_coll_shard5_replica_n11, dataDir=/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/tmp/tests-tmp/solr.cloud.ReplaceNodeTest_AF6A214F1A34E7EE-001/tempDir-001/node4/replacenodetest_coll_shard5_replica_n11/data/, config=solrconfig.xml, schema=schema.xml, startTime=Sun Oct 18 10:09:52 GMT 2020, uptime=17198, lastPublished=active, configVersion=0, cloud={collection=replacenodetest_coll, shard=shard5, replica=core_node12, replicaType=NRT}, index={numDocs=0, maxDoc=0, deletedDocs=0, indexHeapUsageBytes=0, version=3, segmentCount=0, current=false, hasDeletions=false, directory=org.apache.lucene.store.MockDirectoryWrapper:MockDirectoryWrapper(NRTCachingDirectory(ByteBuffersDirectory@569c90f8 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7d9d7617; maxCacheMB=0.5166015625 maxMergeSizeMB=0.4443359375)), segmentsFile=segments_1, segmentsFileSizeInBytes=-1, userData={}, sizeInBytes=135, size=135 bytes}}} expected:<0> but was:<1>

Stack Trace:
java.lang.AssertionError: Expecting no cores but found some: {replacenodetest_coll_shard5_replica_n11={name=replacenodetest_coll_shard5_replica_n11, instanceDir=/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/tmp/tests-tmp/solr.cloud.ReplaceNodeTest_AF6A214F1A34E7EE-001/tempDir-001/node4/replacenodetest_coll_shard5_replica_n11, dataDir=/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/tmp/tests-tmp/solr.cloud.ReplaceNodeTest_AF6A214F1A34E7EE-001/tempDir-001/node4/replacenodetest_coll_shard5_replica_n11/data/, config=solrconfig.xml, schema=schema.xml, startTime=Sun Oct 18 10:09:52 GMT 2020, uptime=17198, lastPublished=active, configVersion=0, cloud={collection=replacenodetest_coll, shard=shard5, replica=core_node12, replicaType=NRT}, index={numDocs=0, maxDoc=0, deletedDocs=0, indexHeapUsageBytes=0, version=3, segmentCount=0, current=false, hasDeletions=false, directory=org.apache.lucene.store.MockDirectoryWrapper:MockDirectoryWrapper(NRTCachingDirectory(ByteBuffersDirectory@569c90f8 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7d9d7617; maxCacheMB=0.5166015625 maxMergeSizeMB=0.4443359375)), segmentsFile=segments_1, segmentsFileSizeInBytes=-1, userData={}, sizeInBytes=135, size=135 bytes}}} expected:<0> but was:<1>
	at __randomizedtesting.SeedInfo.seed([AF6A214F1A34E7EE:273E1E95B4C88A16]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.junit.Assert.failNotEquals(Assert.java:835)
	at org.junit.Assert.assertEquals(Assert.java:647)
	at org.apache.solr.cloud.ReplaceNodeTest.test(ReplaceNodeTest.java:148)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:567)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:830)


FAILED:  org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest.test

Error Message:
java.util.concurrent.TimeoutException: Cluster state still in flux after 30 retry intervals.

Stack Trace:
java.util.concurrent.TimeoutException: Cluster state still in flux after 30 retry intervals.
	at __randomizedtesting.SeedInfo.seed([AF6A214F1A34E7EE:273E1E95B4C88A16]:0)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForThingsToLevelOut(AbstractFullDistribZkTestBase.java:1643)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForThingsToLevelOut(AbstractFullDistribZkTestBase.java:1613)
	at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest.test(ChaosMonkeyNothingIsSafeWithPullReplicasTest.java:253)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:567)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1090)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1061)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:830)


FAILED:  org.apache.solr.cloud.MoveReplicaTest.test

Error Message:
java.lang.AssertionError: replica never fully recovered

Stack Trace:
java.lang.AssertionError: replica never fully recovered
	at __randomizedtesting.SeedInfo.seed([AF6A214F1A34E7EE:273E1E95B4C88A16]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.junit.Assert.assertTrue(Assert.java:42)
	at org.apache.solr.cloud.MoveReplicaTest.test(MoveReplicaTest.java:189)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:567)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:830)


FAILED:  org.apache.solr.cloud.BasicDistributedZk2Test.test

Error Message:
java.lang.AssertionError: There are still nodes recovering - waited for 330SECONDS

Stack Trace:
java.lang.AssertionError: There are still nodes recovering - waited for 330SECONDS
	at __randomizedtesting.SeedInfo.seed([AF6A214F1A34E7EE:273E1E95B4C88A16]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:197)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:144)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:139)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:1051)
	at org.apache.solr.cloud.BasicDistributedZk2Test.brindDownShardIndexSomeDocsAndRecover(BasicDistributedZk2Test.java:362)
	at org.apache.solr.cloud.BasicDistributedZk2Test.test(BasicDistributedZk2Test.java:103)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:567)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1090)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1061)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:830)


FAILED:  org.apache.solr.cloud.DeleteReplicaTest.raceConditionOnDeleteAndRegisterReplica

Error Message:
java.lang.RuntimeException: Failed while waiting for active collection
Timeout waiting to see state for collection=raceDeleteReplicaCollection :DocCollection(raceDeleteReplicaCollection/15)={
  "pullReplicas":"0",
  "replicationFactor":"2",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node4":{
          "core":"raceDeleteReplicaCollection_shard1_replica_n2",
          "core_node_name":"core_node4",
          "base_url":"https://127.0.0.1:37293/solr",
          "node_name":"127.0.0.1:37293_solr",
          "state":"recovering",
          "collection":"raceDeleteReplicaCollection",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node6":{
          "core":"raceDeleteReplicaCollection_shard1_replica_n5",
          "leader":"true",
          "core_node_name":"core_node6",
          "base_url":"https://127.0.0.1:35139/solr",
          "node_name":"127.0.0.1:35139_solr",
          "state":"active",
          "collection":"raceDeleteReplicaCollection",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"2",
  "tlogReplicas":"0"}
Live Nodes: [127.0.0.1:34459_solr, 127.0.0.1:35139_solr, 127.0.0.1:37293_solr, 127.0.0.1:44215_solr]
Last available state: DocCollection(raceDeleteReplicaCollection/15)={
  "pullReplicas":"0",
  "replicationFactor":"2",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node4":{
          "core":"raceDeleteReplicaCollection_shard1_replica_n2",
          "core_node_name":"core_node4",
          "base_url":"https://127.0.0.1:37293/solr",
          "node_name":"127.0.0.1:37293_solr",
          "state":"recovering",
          "collection":"raceDeleteReplicaCollection",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node6":{
          "core":"raceDeleteReplicaCollection_shard1_replica_n5",
          "leader":"true",
          "core_node_name":"core_node6",
          "base_url":"https://127.0.0.1:35139/solr",
          "node_name":"127.0.0.1:35139_solr",
          "state":"active",
          "collection":"raceDeleteReplicaCollection",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"2",
  "tlogReplicas":"0"}

Stack Trace:
java.lang.RuntimeException: Failed while waiting for active collection
Timeout waiting to see state for collection=raceDeleteReplicaCollection :DocCollection(raceDeleteReplicaCollection/15)={
  "pullReplicas":"0",
  "replicationFactor":"2",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node4":{
          "core":"raceDeleteReplicaCollection_shard1_replica_n2",
          "core_node_name":"core_node4",
          "base_url":"https://127.0.0.1:37293/solr",
          "node_name":"127.0.0.1:37293_solr",
          "state":"recovering",
          "collection":"raceDeleteReplicaCollection",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node6":{
          "core":"raceDeleteReplicaCollection_shard1_replica_n5",
          "leader":"true",
          "core_node_name":"core_node6",
          "base_url":"https://127.0.0.1:35139/solr",
          "node_name":"127.0.0.1:35139_solr",
          "state":"active",
          "collection":"raceDeleteReplicaCollection",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"2",
  "tlogReplicas":"0"}
Live Nodes: [127.0.0.1:34459_solr, 127.0.0.1:35139_solr, 127.0.0.1:37293_solr, 127.0.0.1:44215_solr]
Last available state: DocCollection(raceDeleteReplicaCollection/15)={
  "pullReplicas":"0",
  "replicationFactor":"2",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node4":{
          "core":"raceDeleteReplicaCollection_shard1_replica_n2",
          "core_node_name":"core_node4",
          "base_url":"https://127.0.0.1:37293/solr",
          "node_name":"127.0.0.1:37293_solr",
          "state":"recovering",
          "collection":"raceDeleteReplicaCollection",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node6":{
          "core":"raceDeleteReplicaCollection_shard1_replica_n5",
          "leader":"true",
          "core_node_name":"core_node6",
          "base_url":"https://127.0.0.1:35139/solr",
          "node_name":"127.0.0.1:35139_solr",
          "state":"active",
          "collection":"raceDeleteReplicaCollection",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"2",
  "tlogReplicas":"0"}
	at __randomizedtesting.SeedInfo.seed([AF6A214F1A34E7EE:C57C409F72C6AD24]:0)
	at org.apache.solr.cloud.MiniSolrCloudCluster.waitForActiveCollection(MiniSolrCloudCluster.java:774)
	at org.apache.solr.cloud.MiniSolrCloudCluster.waitForActiveCollection(MiniSolrCloudCluster.java:780)
	at org.apache.solr.cloud.DeleteReplicaTest.raceConditionOnDeleteAndRegisterReplica(DeleteReplicaTest.java:385)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:567)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:830)


FAILED:  org.apache.solr.cloud.HttpPartitionTest.test

Error Message:
java.lang.AssertionError: Didn't see all replicas for shard shard1 in collDoRecoveryOnRestart come up within 130000 ms! ClusterState: {
  "control_collection":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{"core_node2":{
            "core":"control_collection_shard1_replica_n1",
            "leader":"true",
            "core_node_name":"core_node2",
            "base_url":"http://127.0.0.1:43541/v/yu",
            "node_name":"127.0.0.1:43541_v%2Fyu",
            "state":"active",
            "collection":"control_collection",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"},
  "collDoRecoveryOnRestart":{
    "pullReplicas":"0",
    "replicationFactor":"2",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"collDoRecoveryOnRestart_shard1_replica_n1",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:35697/v/yu",
            "node_name":"127.0.0.1:35697_v%2Fyu",
            "state":"recovering",
            "collection":"collDoRecoveryOnRestart",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node4":{
            "core":"collDoRecoveryOnRestart_shard1_replica_n2",
            "leader":"true",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:43541/v/yu",
            "node_name":"127.0.0.1:43541_v%2Fyu",
            "state":"active",
            "collection":"collDoRecoveryOnRestart",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"2",
    "tlogReplicas":"0"},
  "collection1":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{
      "shard1":{
        "range":"80000000-ffffffff",
        "state":"active",
        "replicas":{"core_node4":{
            "core":"collection1_shard1_replica_n2",
            "leader":"true",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:45497/v/yu",
            "node_name":"127.0.0.1:45497_v%2Fyu",
            "state":"active",
            "collection":"collection1",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}},
      "shard2":{
        "range":"0-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"collection1_shard2_replica_n1",
            "leader":"true",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:32987/v/yu",
            "node_name":"127.0.0.1:32987_v%2Fyu",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node6":{
            "core":"collection1_shard2_replica_n5",
            "core_node_name":"core_node6",
            "base_url":"http://127.0.0.1:35697/v/yu",
            "node_name":"127.0.0.1:35697_v%2Fyu",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"}}

Stack Trace:
java.lang.AssertionError: Didn't see all replicas for shard shard1 in collDoRecoveryOnRestart come up within 130000 ms! ClusterState: {
  "control_collection":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{"core_node2":{
            "core":"control_collection_shard1_replica_n1",
            "leader":"true",
            "core_node_name":"core_node2",
            "base_url":"http://127.0.0.1:43541/v/yu",
            "node_name":"127.0.0.1:43541_v%2Fyu",
            "state":"active",
            "collection":"control_collection",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"},
  "collDoRecoveryOnRestart":{
    "pullReplicas":"0",
    "replicationFactor":"2",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"collDoRecoveryOnRestart_shard1_replica_n1",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:35697/v/yu",
            "node_name":"127.0.0.1:35697_v%2Fyu",
            "state":"recovering",
            "collection":"collDoRecoveryOnRestart",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node4":{
            "core":"collDoRecoveryOnRestart_shard1_replica_n2",
            "leader":"true",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:43541/v/yu",
            "node_name":"127.0.0.1:43541_v%2Fyu",
            "state":"active",
            "collection":"collDoRecoveryOnRestart",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"2",
    "tlogReplicas":"0"},
  "collection1":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{
      "shard1":{
        "range":"80000000-ffffffff",
        "state":"active",
        "replicas":{"core_node4":{
            "core":"collection1_shard1_replica_n2",
            "leader":"true",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:45497/v/yu",
            "node_name":"127.0.0.1:45497_v%2Fyu",
            "state":"active",
            "collection":"collection1",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}},
      "shard2":{
        "range":"0-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"collection1_shard2_replica_n1",
            "leader":"true",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:32987/v/yu",
            "node_name":"127.0.0.1:32987_v%2Fyu",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node6":{
            "core":"collection1_shard2_replica_n5",
            "core_node_name":"core_node6",
            "base_url":"http://127.0.0.1:35697/v/yu",
            "node_name":"127.0.0.1:35697_v%2Fyu",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"}}
	at __randomizedtesting.SeedInfo.seed([AF6A214F1A34E7EE:273E1E95B4C88A16]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.ensureAllReplicasAreActive(AbstractFullDistribZkTestBase.java:2147)
	at org.apache.solr.cloud.HttpPartitionTest.testDoRecoveryOnRestart(HttpPartitionTest.java:200)
	at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:132)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:567)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1090)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1061)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:830)


FAILED:  org.apache.solr.cloud.HttpPartitionWithTlogReplicasTest.test

Error Message:
java.lang.AssertionError: Didn't see all replicas for shard shard1 in collDoRecoveryOnRestart come up within 130000 ms! ClusterState: {
  "control_collection":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{"core_node2":{
            "core":"control_collection_shard1_replica_n1",
            "leader":"true",
            "core_node_name":"core_node2",
            "base_url":"http://127.0.0.1:35257/nq",
            "node_name":"127.0.0.1:35257_nq",
            "state":"active",
            "collection":"control_collection",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"},
  "collDoRecoveryOnRestart":{
    "pullReplicas":"0",
    "replicationFactor":"0",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"collDoRecoveryOnRestart_shard1_replica_t1",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:46687/nq",
            "node_name":"127.0.0.1:46687_nq",
            "state":"recovering",
            "collection":"collDoRecoveryOnRestart",
            "shard":"shard1",
            "type":"TLOG",
            "force_set_state":"false"},
          "core_node4":{
            "core":"collDoRecoveryOnRestart_shard1_replica_t2",
            "leader":"true",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:34631/nq",
            "node_name":"127.0.0.1:34631_nq",
            "state":"active",
            "collection":"collDoRecoveryOnRestart",
            "shard":"shard1",
            "type":"TLOG",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"0",
    "tlogReplicas":"2"},
  "collection1":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{
      "shard1":{
        "range":"80000000-ffffffff",
        "state":"active",
        "replicas":{"core_node4":{
            "core":"collection1_shard1_replica_t2",
            "leader":"true",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:46687/nq",
            "node_name":"127.0.0.1:46687_nq",
            "state":"active",
            "collection":"collection1",
            "shard":"shard1",
            "type":"TLOG",
            "force_set_state":"false"}}},
      "shard2":{
        "range":"0-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"collection1_shard2_replica_t1",
            "leader":"true",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:34853/nq",
            "node_name":"127.0.0.1:34853_nq",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"TLOG",
            "force_set_state":"false"},
          "core_node6":{
            "core":"collection1_shard2_replica_t5",
            "core_node_name":"core_node6",
            "base_url":"http://127.0.0.1:34631/nq",
            "node_name":"127.0.0.1:34631_nq",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"TLOG",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"}}

Stack Trace:
java.lang.AssertionError: Didn't see all replicas for shard shard1 in collDoRecoveryOnRestart come up within 130000 ms! ClusterState: {
  "control_collection":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{"core_node2":{
            "core":"control_collection_shard1_replica_n1",
            "leader":"true",
            "core_node_name":"core_node2",
            "base_url":"http://127.0.0.1:35257/nq",
            "node_name":"127.0.0.1:35257_nq",
            "state":"active",
            "collection":"control_collection",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"},
  "collDoRecoveryOnRestart":{
    "pullReplicas":"0",
    "replicationFactor":"0",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"collDoRecoveryOnRestart_shard1_replica_t1",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:46687/nq",
            "node_name":"127.0.0.1:46687_nq",
            "state":"recovering",
            "collection":"collDoRecoveryOnRestart",
            "shard":"shard1",
            "type":"TLOG",
            "force_set_state":"false"},
          "core_node4":{
            "core":"collDoRecoveryOnRestart_shard1_replica_t2",
            "leader":"true",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:34631/nq",
            "node_name":"127.0.0.1:34631_nq",
            "state":"active",
            "collection":"collDoRecoveryOnRestart",
            "shard":"shard1",
            "type":"TLOG",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"0",
    "tlogReplicas":"2"},
  "collection1":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{
      "shard1":{
        "range":"80000000-ffffffff",
        "state":"active",
        "replicas":{"core_node4":{
            "core":"collection1_shard1_replica_t2",
            "leader":"true",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:46687/nq",
            "node_name":"127.0.0.1:46687_nq",
            "state":"active",
            "collection":"collection1",
            "shard":"shard1",
            "type":"TLOG",
            "force_set_state":"false"}}},
      "shard2":{
        "range":"0-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"collection1_shard2_replica_t1",
            "leader":"true",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:34853/nq",
            "node_name":"127.0.0.1:34853_nq",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"TLOG",
            "force_set_state":"false"},
          "core_node6":{
            "core":"collection1_shard2_replica_t5",
            "core_node_name":"core_node6",
            "base_url":"http://127.0.0.1:34631/nq",
            "node_name":"127.0.0.1:34631_nq",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"TLOG",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"}}
	at __randomizedtesting.SeedInfo.seed([AF6A214F1A34E7EE:273E1E95B4C88A16]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.ensureAllReplicasAreActive(AbstractFullDistribZkTestBase.java:2147)
	at org.apache.solr.cloud.HttpPartitionTest.testDoRecoveryOnRestart(HttpPartitionTest.java:200)
	at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:132)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:567)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1090)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1061)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:830)


FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test

Error Message:
java.lang.AssertionError: Didn't see all replicas for shard shard1 in c8n_1x3_lf come up within 90000 ms! ClusterState: {
  "control_collection":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{"core_node2":{
            "core":"control_collection_shard1_replica_n1",
            "leader":"true",
            "core_node_name":"core_node2",
            "base_url":"http://127.0.0.1:35443/b_cwg",
            "node_name":"127.0.0.1:35443_b_cwg",
            "state":"active",
            "collection":"control_collection",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"},
  "c8n_1x3_lf":{
    "pullReplicas":"0",
    "replicationFactor":"3",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"c8n_1x3_lf_shard1_replica_n1",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:37607/b_cwg",
            "node_name":"127.0.0.1:37607_b_cwg",
            "state":"recovering",
            "collection":"c8n_1x3_lf",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node5":{
            "core":"c8n_1x3_lf_shard1_replica_n2",
            "leader":"true",
            "core_node_name":"core_node5",
            "base_url":"http://127.0.0.1:38325/b_cwg",
            "node_name":"127.0.0.1:38325_b_cwg",
            "state":"active",
            "collection":"c8n_1x3_lf",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node6":{
            "core":"c8n_1x3_lf_shard1_replica_n4",
            "core_node_name":"core_node6",
            "base_url":"http://127.0.0.1:35443/b_cwg",
            "node_name":"127.0.0.1:35443_b_cwg",
            "state":"recovering",
            "collection":"c8n_1x3_lf",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"3",
    "tlogReplicas":"0"},
  "collection1":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{
      "shard1":{
        "range":"80000000-ffffffff",
        "state":"active",
        "replicas":{"core_node4":{
            "core":"collection1_shard1_replica_n2",
            "leader":"true",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:37607/b_cwg",
            "node_name":"127.0.0.1:37607_b_cwg",
            "state":"active",
            "collection":"collection1",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}},
      "shard2":{
        "range":"0-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"collection1_shard2_replica_n1",
            "leader":"true",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:32923/b_cwg",
            "node_name":"127.0.0.1:32923_b_cwg",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node6":{
            "core":"collection1_shard2_replica_n5",
            "core_node_name":"core_node6",
            "base_url":"http://127.0.0.1:38325/b_cwg",
            "node_name":"127.0.0.1:38325_b_cwg",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"}}

Stack Trace:
java.lang.AssertionError: Didn't see all replicas for shard shard1 in c8n_1x3_lf come up within 90000 ms! ClusterState: {
  "control_collection":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{"core_node2":{
            "core":"control_collection_shard1_replica_n1",
            "leader":"true",
            "core_node_name":"core_node2",
            "base_url":"http://127.0.0.1:35443/b_cwg",
            "node_name":"127.0.0.1:35443_b_cwg",
            "state":"active",
            "collection":"control_collection",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"},
  "c8n_1x3_lf":{
    "pullReplicas":"0",
    "replicationFactor":"3",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"c8n_1x3_lf_shard1_replica_n1",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:37607/b_cwg",
            "node_name":"127.0.0.1:37607_b_cwg",
            "state":"recovering",
            "collection":"c8n_1x3_lf",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node5":{
            "core":"c8n_1x3_lf_shard1_replica_n2",
            "leader":"true",
            "core_node_name":"core_node5",
            "base_url":"http://127.0.0.1:38325/b_cwg",
            "node_name":"127.0.0.1:38325_b_cwg",
            "state":"active",
            "collection":"c8n_1x3_lf",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node6":{
            "core":"c8n_1x3_lf_shard1_replica_n4",
            "core_node_name":"core_node6",
            "base_url":"http://127.0.0.1:35443/b_cwg",
            "node_name":"127.0.0.1:35443_b_cwg",
            "state":"recovering",
            "collection":"c8n_1x3_lf",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"3",
    "tlogReplicas":"0"},
  "collection1":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{
      "shard1":{
        "range":"80000000-ffffffff",
        "state":"active",
        "replicas":{"core_node4":{
            "core":"collection1_shard1_replica_n2",
            "leader":"true",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:37607/b_cwg",
            "node_name":"127.0.0.1:37607_b_cwg",
            "state":"active",
            "collection":"collection1",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}},
      "shard2":{
        "range":"0-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"collection1_shard2_replica_n1",
            "leader":"true",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:32923/b_cwg",
            "node_name":"127.0.0.1:32923_b_cwg",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node6":{
            "core":"collection1_shard2_replica_n5",
            "core_node_name":"core_node6",
            "base_url":"http://127.0.0.1:38325/b_cwg",
            "node_name":"127.0.0.1:38325_b_cwg",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"}}
	at __randomizedtesting.SeedInfo.seed([AF6A214F1A34E7EE:273E1E95B4C88A16]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.ensureAllReplicasAreActive(AbstractFullDistribZkTestBase.java:2147)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:101)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:58)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:567)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1090)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1061)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:830)


FAILED:  org.apache.solr.cloud.LeaderFailureAfterFreshStartTest.test

Error Message:
java.lang.AssertionError: timeout waiting to see all nodes active

Stack Trace:
java.lang.AssertionError: timeout waiting to see all nodes active
	at __randomizedtesting.SeedInfo.seed([AF6A214F1A34E7EE:273E1E95B4C88A16]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.solr.cloud.LeaderFailureAfterFreshStartTest.waitTillNodesActive(LeaderFailureAfterFreshStartTest.java:234)
	at org.apache.solr.cloud.LeaderFailureAfterFreshStartTest.restartNodes(LeaderFailureAfterFreshStartTest.java:174)
	at org.apache.solr.cloud.LeaderFailureAfterFreshStartTest.test(LeaderFailureAfterFreshStartTest.java:149)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:567)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1090)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1061)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRule

[...truncated too long message...]

=false -Ptests.file.encoding=ISO-8859-1

  - org.apache.solr.cloud.SyncSliceTest.test (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.SyncSliceTest.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.SyncSliceTest" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseSerialGC" -Ptests.seed=AF6A214F1A34E7EE -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=ISO-8859-1

  - org.apache.solr.cloud.UnloadDistributedZkTest.test (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.UnloadDistributedZkTest.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.UnloadDistributedZkTest" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseSerialGC" -Ptests.seed=AF6A214F1A34E7EE -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=ISO-8859-1

  - org.apache.solr.cloud.TestCloudRecovery.corruptedLogTest (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.TestCloudRecovery.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.TestCloudRecovery" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseSerialGC" -Ptests.seed=AF6A214F1A34E7EE -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=ISO-8859-1

  - org.apache.solr.cloud.MigrateRouteKeyTest.multipleShardMigrateTest (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.MigrateRouteKeyTest.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.MigrateRouteKeyTest" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseSerialGC" -Ptests.seed=AF6A214F1A34E7EE -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=ISO-8859-1

  - org.apache.solr.cloud.ReplaceNodeTest.test (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.ReplaceNodeTest.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.ReplaceNodeTest" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseSerialGC" -Ptests.seed=AF6A214F1A34E7EE -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=ISO-8859-1

  - org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.RecoveryAfterSoftCommitTest.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.RecoveryAfterSoftCommitTest" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseSerialGC" -Ptests.seed=AF6A214F1A34E7EE -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=ISO-8859-1

  - org.apache.solr.cloud.DeleteReplicaTest.raceConditionOnDeleteAndRegisterReplica (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.DeleteReplicaTest.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.DeleteReplicaTest" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseSerialGC" -Ptests.seed=AF6A214F1A34E7EE -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=ISO-8859-1

  - org.apache.solr.cloud.TestLeaderElectionWithEmptyReplica.test (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.TestLeaderElectionWithEmptyReplica.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.TestLeaderElectionWithEmptyReplica" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseSerialGC" -Ptests.seed=AF6A214F1A34E7EE -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=ISO-8859-1

  - org.apache.solr.core.snapshots.TestSolrCloudSnapshots.testSnapshots (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.core.snapshots.TestSolrCloudSnapshots.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.core.snapshots.TestSolrCloudSnapshots" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseSerialGC" -Ptests.seed=AF6A214F1A34E7EE -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=ISO-8859-1

  - org.apache.solr.cloud.MoveReplicaTest.test (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.MoveReplicaTest.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.MoveReplicaTest" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseSerialGC" -Ptests.seed=AF6A214F1A34E7EE -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=ISO-8859-1

  - org.apache.solr.cloud.RecoveryZkTest.test (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.RecoveryZkTest.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.RecoveryZkTest" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseSerialGC" -Ptests.seed=AF6A214F1A34E7EE -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=ISO-8859-1

  - org.apache.lucene.search.spans.TestSpanExplanationsOfNonMatches.testExplainWithoutScoring (:lucene:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/lucene/core/build/test-results/test/outputs/OUTPUT-org.apache.lucene.search.spans.TestSpanExplanationsOfNonMatches.txt
    Reproduce with: gradlew :lucene:core:test --tests "org.apache.lucene.search.spans.TestSpanExplanationsOfNonMatches" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseSerialGC" -Ptests.seed=AF6A214F1A34E7EE -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=ISO-8859-1

  - org.apache.lucene.search.spans.TestSpanExplanations.testExplainWithoutScoring (:lucene:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/lucene/core/build/test-results/test/outputs/OUTPUT-org.apache.lucene.search.spans.TestSpanExplanations.txt
    Reproduce with: gradlew :lucene:core:test --tests "org.apache.lucene.search.spans.TestSpanExplanations" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseSerialGC" -Ptests.seed=AF6A214F1A34E7EE -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=ISO-8859-1


Deprecated Gradle features were used in this build, making it incompatible with Gradle 7.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/6.6.1/userguide/command_line_interface.html#sec:command_line_warnings

BUILD SUCCESSFUL in 59m 58s
854 actionable tasks: 854 executed
Build step 'Invoke Gradle script' changed build result to SUCCESS
Archiving artifacts
java.lang.InterruptedException: no matches found within 10000
	at hudson.FilePath$ValidateAntFileMask.hasMatch(FilePath.java:2875)
	at hudson.FilePath$ValidateAntFileMask.invoke(FilePath.java:2754)
	at hudson.FilePath$ValidateAntFileMask.invoke(FilePath.java:2735)
	at hudson.FilePath.act(FilePath.java:1076)
	at hudson.FilePath.act(FilePath.java:1059)
	at hudson.FilePath.validateAntFileMask(FilePath.java:2733)
	at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:270)
	at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:80)
	at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
	at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:804)
	at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:753)
	at hudson.model.Build$BuildExecution.post2(Build.java:177)
	at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:698)
	at hudson.model.Run.execute(Run.java:1919)
	at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
	at hudson.model.ResourceController.execute(ResourceController.java:97)
	at hudson.model.Executor.run(Executor.java:428)
No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/*_pid*.log". Configuration error?
[Java] Sleeping for 5 seconds due to JENKINS-32191...
[Java] Parsing console log (workspace: '/home/jenkins/workspace/Lucene-Solr-master-Linux')
[Java] Successfully parsed console log
[Java] -> found 0 issues (skipped 0 duplicates)
[Java] Skipping post processing
[Java] No filter has been set, publishing all 0 issues
[Java] Using reference build 'Lucene-Solr-master-Linux #28361' to compute new, fixed, and outstanding issues
[Java] Issues delta (vs. reference build): outstanding: 0, new: 0, fixed: 0
[Java] No quality gates have been set - skipping
[Java] Health report is disabled - skipping
[Java] Created analysis result for 0 issues (found 0 new issues, fixed 0 issues)
[Java] Attaching ResultAction with ID 'java' to run 'Lucene-Solr-master-Linux #28362'.
[Checks API] No suitable checks publisher found.
Recording test results
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
[Checks API] No suitable checks publisher found.
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)

[JENKINS] Lucene-Solr-master-Linux (64bit/jdk-14.0.1) - Build # 28361 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Linux/28361/
Java: 64bit/jdk-14.0.1 -XX:+UseCompressedOops -XX:+UseSerialGC

51 tests failed.
FAILED:  org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test

Error Message:
java.lang.AssertionError: There are still nodes recovering - waited for 180SECONDS

Stack Trace:
java.lang.AssertionError: There are still nodes recovering - waited for 180SECONDS
	at __randomizedtesting.SeedInfo.seed([5FF4BF6FD205CC18:D7A080B57CF9A1E0]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:197)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:150)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:1063)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForThingsToLevelOut(AbstractFullDistribZkTestBase.java:1621)
	at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test(ChaosMonkeySafeLeaderTest.java:156)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:564)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1090)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1061)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:832)


FAILED:  org.apache.solr.cloud.ReplaceNodeTest.test

Error Message:
java.lang.AssertionError: Expecting no cores but found some: {replacenodetest_coll_shard1_replica_n11={name=replacenodetest_coll_shard1_replica_n11, instanceDir=/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/tmp/tests-tmp/solr.cloud.ReplaceNodeTest_5FF4BF6FD205CC18-001/tempDir-001/node4/replacenodetest_coll_shard1_replica_n11, dataDir=/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/tmp/tests-tmp/solr.cloud.ReplaceNodeTest_5FF4BF6FD205CC18-001/tempDir-001/node4/replacenodetest_coll_shard1_replica_n11/data/, config=solrconfig.xml, schema=schema.xml, startTime=Sun Oct 18 02:41:49 GMT 2020, uptime=17357, lastPublished=active, configVersion=0, cloud={collection=replacenodetest_coll, shard=shard1, replica=core_node12, replicaType=NRT}, index={numDocs=0, maxDoc=0, deletedDocs=0, indexHeapUsageBytes=0, version=3, segmentCount=0, current=false, hasDeletions=false, directory=org.apache.lucene.store.MockDirectoryWrapper:MockDirectoryWrapper(ByteBuffersDirectory@120961ce lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@60883795), segmentsFile=segments_1, segmentsFileSizeInBytes=-1, userData={}, sizeInBytes=135, size=135 bytes}}} expected:<0> but was:<1>

Stack Trace:
java.lang.AssertionError: Expecting no cores but found some: {replacenodetest_coll_shard1_replica_n11={name=replacenodetest_coll_shard1_replica_n11, instanceDir=/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/tmp/tests-tmp/solr.cloud.ReplaceNodeTest_5FF4BF6FD205CC18-001/tempDir-001/node4/replacenodetest_coll_shard1_replica_n11, dataDir=/home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/tmp/tests-tmp/solr.cloud.ReplaceNodeTest_5FF4BF6FD205CC18-001/tempDir-001/node4/replacenodetest_coll_shard1_replica_n11/data/, config=solrconfig.xml, schema=schema.xml, startTime=Sun Oct 18 02:41:49 GMT 2020, uptime=17357, lastPublished=active, configVersion=0, cloud={collection=replacenodetest_coll, shard=shard1, replica=core_node12, replicaType=NRT}, index={numDocs=0, maxDoc=0, deletedDocs=0, indexHeapUsageBytes=0, version=3, segmentCount=0, current=false, hasDeletions=false, directory=org.apache.lucene.store.MockDirectoryWrapper:MockDirectoryWrapper(ByteBuffersDirectory@120961ce lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@60883795), segmentsFile=segments_1, segmentsFileSizeInBytes=-1, userData={}, sizeInBytes=135, size=135 bytes}}} expected:<0> but was:<1>
	at __randomizedtesting.SeedInfo.seed([5FF4BF6FD205CC18:D7A080B57CF9A1E0]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.junit.Assert.failNotEquals(Assert.java:835)
	at org.junit.Assert.assertEquals(Assert.java:647)
	at org.apache.solr.cloud.ReplaceNodeTest.test(ReplaceNodeTest.java:148)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:564)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:832)


FAILED:  org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest.test

Error Message:
java.util.concurrent.TimeoutException: Cluster state still in flux after 30 retry intervals.

Stack Trace:
java.util.concurrent.TimeoutException: Cluster state still in flux after 30 retry intervals.
	at __randomizedtesting.SeedInfo.seed([5FF4BF6FD205CC18:D7A080B57CF9A1E0]:0)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForThingsToLevelOut(AbstractFullDistribZkTestBase.java:1643)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForThingsToLevelOut(AbstractFullDistribZkTestBase.java:1613)
	at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest.test(ChaosMonkeyNothingIsSafeWithPullReplicasTest.java:253)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:564)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1090)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1061)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:832)


FAILED:  org.apache.solr.cloud.MoveReplicaTest.test

Error Message:
java.lang.AssertionError: replica never fully recovered

Stack Trace:
java.lang.AssertionError: replica never fully recovered
	at __randomizedtesting.SeedInfo.seed([5FF4BF6FD205CC18:D7A080B57CF9A1E0]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.junit.Assert.assertTrue(Assert.java:42)
	at org.apache.solr.cloud.MoveReplicaTest.test(MoveReplicaTest.java:189)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:564)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:832)


FAILED:  org.apache.solr.cloud.BasicDistributedZk2Test.test

Error Message:
java.lang.AssertionError: There are still nodes recovering - waited for 330SECONDS

Stack Trace:
java.lang.AssertionError: There are still nodes recovering - waited for 330SECONDS
	at __randomizedtesting.SeedInfo.seed([5FF4BF6FD205CC18:D7A080B57CF9A1E0]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:197)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:144)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:139)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:1051)
	at org.apache.solr.cloud.BasicDistributedZk2Test.brindDownShardIndexSomeDocsAndRecover(BasicDistributedZk2Test.java:362)
	at org.apache.solr.cloud.BasicDistributedZk2Test.test(BasicDistributedZk2Test.java:103)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:564)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1090)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1061)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:832)


FAILED:  org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest.test

Error Message:
java.lang.AssertionError: There are still nodes recovering - waited for 180SECONDS

Stack Trace:
java.lang.AssertionError: There are still nodes recovering - waited for 180SECONDS
	at __randomizedtesting.SeedInfo.seed([5FF4BF6FD205CC18:D7A080B57CF9A1E0]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:197)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:150)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:1063)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForThingsToLevelOut(AbstractFullDistribZkTestBase.java:1621)
	at org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest.test(ChaosMonkeySafeLeaderWithPullReplicasTest.java:198)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:564)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1090)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1061)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:832)


FAILED:  org.apache.solr.cloud.DeleteReplicaTest.raceConditionOnDeleteAndRegisterReplica

Error Message:
java.lang.RuntimeException: Failed while waiting for active collection
Timeout waiting to see state for collection=raceDeleteReplicaCollection :DocCollection(raceDeleteReplicaCollection/16)={
  "pullReplicas":"0",
  "replicationFactor":"2",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node3":{
          "core":"raceDeleteReplicaCollection_shard1_replica_n1",
          "core_node_name":"core_node3",
          "base_url":"https://127.0.0.1:40037/solr",
          "node_name":"127.0.0.1:40037_solr",
          "state":"recovering",
          "collection":"raceDeleteReplicaCollection",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node6":{
          "core":"raceDeleteReplicaCollection_shard1_replica_n5",
          "leader":"true",
          "core_node_name":"core_node6",
          "base_url":"https://127.0.0.1:37355/solr",
          "node_name":"127.0.0.1:37355_solr",
          "state":"active",
          "collection":"raceDeleteReplicaCollection",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"2",
  "tlogReplicas":"0"}
Live Nodes: [127.0.0.1:37355_solr, 127.0.0.1:40037_solr, 127.0.0.1:44323_solr, 127.0.0.1:44811_solr]
Last available state: DocCollection(raceDeleteReplicaCollection/16)={
  "pullReplicas":"0",
  "replicationFactor":"2",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node3":{
          "core":"raceDeleteReplicaCollection_shard1_replica_n1",
          "core_node_name":"core_node3",
          "base_url":"https://127.0.0.1:40037/solr",
          "node_name":"127.0.0.1:40037_solr",
          "state":"recovering",
          "collection":"raceDeleteReplicaCollection",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node6":{
          "core":"raceDeleteReplicaCollection_shard1_replica_n5",
          "leader":"true",
          "core_node_name":"core_node6",
          "base_url":"https://127.0.0.1:37355/solr",
          "node_name":"127.0.0.1:37355_solr",
          "state":"active",
          "collection":"raceDeleteReplicaCollection",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"2",
  "tlogReplicas":"0"}

Stack Trace:
java.lang.RuntimeException: Failed while waiting for active collection
Timeout waiting to see state for collection=raceDeleteReplicaCollection :DocCollection(raceDeleteReplicaCollection/16)={
  "pullReplicas":"0",
  "replicationFactor":"2",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node3":{
          "core":"raceDeleteReplicaCollection_shard1_replica_n1",
          "core_node_name":"core_node3",
          "base_url":"https://127.0.0.1:40037/solr",
          "node_name":"127.0.0.1:40037_solr",
          "state":"recovering",
          "collection":"raceDeleteReplicaCollection",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node6":{
          "core":"raceDeleteReplicaCollection_shard1_replica_n5",
          "leader":"true",
          "core_node_name":"core_node6",
          "base_url":"https://127.0.0.1:37355/solr",
          "node_name":"127.0.0.1:37355_solr",
          "state":"active",
          "collection":"raceDeleteReplicaCollection",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"2",
  "tlogReplicas":"0"}
Live Nodes: [127.0.0.1:37355_solr, 127.0.0.1:40037_solr, 127.0.0.1:44323_solr, 127.0.0.1:44811_solr]
Last available state: DocCollection(raceDeleteReplicaCollection/16)={
  "pullReplicas":"0",
  "replicationFactor":"2",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node3":{
          "core":"raceDeleteReplicaCollection_shard1_replica_n1",
          "core_node_name":"core_node3",
          "base_url":"https://127.0.0.1:40037/solr",
          "node_name":"127.0.0.1:40037_solr",
          "state":"recovering",
          "collection":"raceDeleteReplicaCollection",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node6":{
          "core":"raceDeleteReplicaCollection_shard1_replica_n5",
          "leader":"true",
          "core_node_name":"core_node6",
          "base_url":"https://127.0.0.1:37355/solr",
          "node_name":"127.0.0.1:37355_solr",
          "state":"active",
          "collection":"raceDeleteReplicaCollection",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"2",
  "tlogReplicas":"0"}
	at __randomizedtesting.SeedInfo.seed([5FF4BF6FD205CC18:35E2DEBFBAF786D2]:0)
	at org.apache.solr.cloud.MiniSolrCloudCluster.waitForActiveCollection(MiniSolrCloudCluster.java:774)
	at org.apache.solr.cloud.MiniSolrCloudCluster.waitForActiveCollection(MiniSolrCloudCluster.java:780)
	at org.apache.solr.cloud.DeleteReplicaTest.raceConditionOnDeleteAndRegisterReplica(DeleteReplicaTest.java:385)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:564)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:832)


FAILED:  org.apache.solr.cloud.HttpPartitionTest.test

Error Message:
java.lang.AssertionError: Didn't see all replicas for shard shard1 in collDoRecoveryOnRestart come up within 130000 ms! ClusterState: {
  "control_collection":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{"core_node2":{
            "core":"control_collection_shard1_replica_n1",
            "leader":"true",
            "core_node_name":"core_node2",
            "base_url":"http://127.0.0.1:36371/dvbw",
            "node_name":"127.0.0.1:36371_dvbw",
            "state":"active",
            "collection":"control_collection",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"},
  "collDoRecoveryOnRestart":{
    "pullReplicas":"0",
    "replicationFactor":"2",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"collDoRecoveryOnRestart_shard1_replica_n1",
            "leader":"true",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:46105/dvbw",
            "node_name":"127.0.0.1:46105_dvbw",
            "state":"active",
            "collection":"collDoRecoveryOnRestart",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node4":{
            "core":"collDoRecoveryOnRestart_shard1_replica_n2",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:42067/dvbw",
            "node_name":"127.0.0.1:42067_dvbw",
            "state":"recovering",
            "collection":"collDoRecoveryOnRestart",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"2",
    "tlogReplicas":"0"},
  "collection1":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{
      "shard1":{
        "range":"80000000-ffffffff",
        "state":"active",
        "replicas":{"core_node4":{
            "core":"collection1_shard1_replica_n2",
            "leader":"true",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:42067/dvbw",
            "node_name":"127.0.0.1:42067_dvbw",
            "state":"active",
            "collection":"collection1",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}},
      "shard2":{
        "range":"0-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"collection1_shard2_replica_n1",
            "leader":"true",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:35719/dvbw",
            "node_name":"127.0.0.1:35719_dvbw",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node6":{
            "core":"collection1_shard2_replica_n5",
            "core_node_name":"core_node6",
            "base_url":"http://127.0.0.1:46105/dvbw",
            "node_name":"127.0.0.1:46105_dvbw",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"}}

Stack Trace:
java.lang.AssertionError: Didn't see all replicas for shard shard1 in collDoRecoveryOnRestart come up within 130000 ms! ClusterState: {
  "control_collection":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{"core_node2":{
            "core":"control_collection_shard1_replica_n1",
            "leader":"true",
            "core_node_name":"core_node2",
            "base_url":"http://127.0.0.1:36371/dvbw",
            "node_name":"127.0.0.1:36371_dvbw",
            "state":"active",
            "collection":"control_collection",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"},
  "collDoRecoveryOnRestart":{
    "pullReplicas":"0",
    "replicationFactor":"2",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"collDoRecoveryOnRestart_shard1_replica_n1",
            "leader":"true",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:46105/dvbw",
            "node_name":"127.0.0.1:46105_dvbw",
            "state":"active",
            "collection":"collDoRecoveryOnRestart",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node4":{
            "core":"collDoRecoveryOnRestart_shard1_replica_n2",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:42067/dvbw",
            "node_name":"127.0.0.1:42067_dvbw",
            "state":"recovering",
            "collection":"collDoRecoveryOnRestart",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"2",
    "tlogReplicas":"0"},
  "collection1":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{
      "shard1":{
        "range":"80000000-ffffffff",
        "state":"active",
        "replicas":{"core_node4":{
            "core":"collection1_shard1_replica_n2",
            "leader":"true",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:42067/dvbw",
            "node_name":"127.0.0.1:42067_dvbw",
            "state":"active",
            "collection":"collection1",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}},
      "shard2":{
        "range":"0-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"collection1_shard2_replica_n1",
            "leader":"true",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:35719/dvbw",
            "node_name":"127.0.0.1:35719_dvbw",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node6":{
            "core":"collection1_shard2_replica_n5",
            "core_node_name":"core_node6",
            "base_url":"http://127.0.0.1:46105/dvbw",
            "node_name":"127.0.0.1:46105_dvbw",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"}}
	at __randomizedtesting.SeedInfo.seed([5FF4BF6FD205CC18:D7A080B57CF9A1E0]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.ensureAllReplicasAreActive(AbstractFullDistribZkTestBase.java:2147)
	at org.apache.solr.cloud.HttpPartitionTest.testDoRecoveryOnRestart(HttpPartitionTest.java:200)
	at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:132)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:564)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1090)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1061)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:832)


FAILED:  org.apache.solr.cloud.HttpPartitionWithTlogReplicasTest.test

Error Message:
java.lang.AssertionError: Didn't see all replicas for shard shard1 in collDoRecoveryOnRestart come up within 130000 ms! ClusterState: {
  "control_collection":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{"core_node2":{
            "core":"control_collection_shard1_replica_n1",
            "leader":"true",
            "core_node_name":"core_node2",
            "base_url":"http://127.0.0.1:40145/no/og",
            "node_name":"127.0.0.1:40145_no%2Fog",
            "state":"active",
            "collection":"control_collection",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"},
  "collDoRecoveryOnRestart":{
    "pullReplicas":"0",
    "replicationFactor":"0",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"collDoRecoveryOnRestart_shard1_replica_t1",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:40145/no/og",
            "node_name":"127.0.0.1:40145_no%2Fog",
            "state":"recovering",
            "collection":"collDoRecoveryOnRestart",
            "shard":"shard1",
            "type":"TLOG",
            "force_set_state":"false"},
          "core_node4":{
            "core":"collDoRecoveryOnRestart_shard1_replica_t2",
            "leader":"true",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:33477/no/og",
            "node_name":"127.0.0.1:33477_no%2Fog",
            "state":"active",
            "collection":"collDoRecoveryOnRestart",
            "shard":"shard1",
            "type":"TLOG",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"0",
    "tlogReplicas":"2"},
  "collection1":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{
      "shard1":{
        "range":"80000000-ffffffff",
        "state":"active",
        "replicas":{"core_node4":{
            "core":"collection1_shard1_replica_t2",
            "leader":"true",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:43661/no/og",
            "node_name":"127.0.0.1:43661_no%2Fog",
            "state":"active",
            "collection":"collection1",
            "shard":"shard1",
            "type":"TLOG",
            "force_set_state":"false"}}},
      "shard2":{
        "range":"0-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"collection1_shard2_replica_t1",
            "leader":"true",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:36247/no/og",
            "node_name":"127.0.0.1:36247_no%2Fog",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"TLOG",
            "force_set_state":"false"},
          "core_node6":{
            "core":"collection1_shard2_replica_t5",
            "core_node_name":"core_node6",
            "base_url":"http://127.0.0.1:33477/no/og",
            "node_name":"127.0.0.1:33477_no%2Fog",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"TLOG",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"}}

Stack Trace:
java.lang.AssertionError: Didn't see all replicas for shard shard1 in collDoRecoveryOnRestart come up within 130000 ms! ClusterState: {
  "control_collection":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{"core_node2":{
            "core":"control_collection_shard1_replica_n1",
            "leader":"true",
            "core_node_name":"core_node2",
            "base_url":"http://127.0.0.1:40145/no/og",
            "node_name":"127.0.0.1:40145_no%2Fog",
            "state":"active",
            "collection":"control_collection",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"},
  "collDoRecoveryOnRestart":{
    "pullReplicas":"0",
    "replicationFactor":"0",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"collDoRecoveryOnRestart_shard1_replica_t1",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:40145/no/og",
            "node_name":"127.0.0.1:40145_no%2Fog",
            "state":"recovering",
            "collection":"collDoRecoveryOnRestart",
            "shard":"shard1",
            "type":"TLOG",
            "force_set_state":"false"},
          "core_node4":{
            "core":"collDoRecoveryOnRestart_shard1_replica_t2",
            "leader":"true",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:33477/no/og",
            "node_name":"127.0.0.1:33477_no%2Fog",
            "state":"active",
            "collection":"collDoRecoveryOnRestart",
            "shard":"shard1",
            "type":"TLOG",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"0",
    "tlogReplicas":"2"},
  "collection1":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{
      "shard1":{
        "range":"80000000-ffffffff",
        "state":"active",
        "replicas":{"core_node4":{
            "core":"collection1_shard1_replica_t2",
            "leader":"true",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:43661/no/og",
            "node_name":"127.0.0.1:43661_no%2Fog",
            "state":"active",
            "collection":"collection1",
            "shard":"shard1",
            "type":"TLOG",
            "force_set_state":"false"}}},
      "shard2":{
        "range":"0-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"collection1_shard2_replica_t1",
            "leader":"true",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:36247/no/og",
            "node_name":"127.0.0.1:36247_no%2Fog",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"TLOG",
            "force_set_state":"false"},
          "core_node6":{
            "core":"collection1_shard2_replica_t5",
            "core_node_name":"core_node6",
            "base_url":"http://127.0.0.1:33477/no/og",
            "node_name":"127.0.0.1:33477_no%2Fog",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"TLOG",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"}}
	at __randomizedtesting.SeedInfo.seed([5FF4BF6FD205CC18:D7A080B57CF9A1E0]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.ensureAllReplicasAreActive(AbstractFullDistribZkTestBase.java:2147)
	at org.apache.solr.cloud.HttpPartitionTest.testDoRecoveryOnRestart(HttpPartitionTest.java:200)
	at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:132)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:564)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1090)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1061)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:832)


FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test

Error Message:
java.lang.AssertionError: Didn't see all replicas for shard shard1 in c8n_1x3_lf come up within 90000 ms! ClusterState: {
  "control_collection":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{"core_node2":{
            "core":"control_collection_shard1_replica_n1",
            "leader":"true",
            "core_node_name":"core_node2",
            "base_url":"http://127.0.0.1:42483/v_/gn",
            "node_name":"127.0.0.1:42483_v_%2Fgn",
            "state":"active",
            "collection":"control_collection",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"},
  "c8n_1x3_lf":{
    "pullReplicas":"0",
    "replicationFactor":"3",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"c8n_1x3_lf_shard1_replica_n1",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:42483/v_/gn",
            "node_name":"127.0.0.1:42483_v_%2Fgn",
            "state":"recovering",
            "collection":"c8n_1x3_lf",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node5":{
            "core":"c8n_1x3_lf_shard1_replica_n2",
            "core_node_name":"core_node5",
            "base_url":"http://127.0.0.1:39657/v_/gn",
            "node_name":"127.0.0.1:39657_v_%2Fgn",
            "state":"recovering",
            "collection":"c8n_1x3_lf",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node6":{
            "core":"c8n_1x3_lf_shard1_replica_n4",
            "leader":"true",
            "core_node_name":"core_node6",
            "base_url":"http://127.0.0.1:40379/v_/gn",
            "node_name":"127.0.0.1:40379_v_%2Fgn",
            "state":"active",
            "collection":"c8n_1x3_lf",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"3",
    "tlogReplicas":"0"},
  "collection1":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{
      "shard1":{
        "range":"80000000-ffffffff",
        "state":"active",
        "replicas":{"core_node3":{
            "core":"collection1_shard1_replica_n1",
            "leader":"true",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:39657/v_/gn",
            "node_name":"127.0.0.1:39657_v_%2Fgn",
            "state":"active",
            "collection":"collection1",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}},
      "shard2":{
        "range":"0-7fffffff",
        "state":"active",
        "replicas":{
          "core_node4":{
            "core":"collection1_shard2_replica_n2",
            "leader":"true",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:40379/v_/gn",
            "node_name":"127.0.0.1:40379_v_%2Fgn",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node6":{
            "core":"collection1_shard2_replica_n5",
            "core_node_name":"core_node6",
            "base_url":"http://127.0.0.1:44599/v_/gn",
            "node_name":"127.0.0.1:44599_v_%2Fgn",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"}}

Stack Trace:
java.lang.AssertionError: Didn't see all replicas for shard shard1 in c8n_1x3_lf come up within 90000 ms! ClusterState: {
  "control_collection":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{"core_node2":{
            "core":"control_collection_shard1_replica_n1",
            "leader":"true",
            "core_node_name":"core_node2",
            "base_url":"http://127.0.0.1:42483/v_/gn",
            "node_name":"127.0.0.1:42483_v_%2Fgn",
            "state":"active",
            "collection":"control_collection",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"},
  "c8n_1x3_lf":{
    "pullReplicas":"0",
    "replicationFactor":"3",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"c8n_1x3_lf_shard1_replica_n1",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:42483/v_/gn",
            "node_name":"127.0.0.1:42483_v_%2Fgn",
            "state":"recovering",
            "collection":"c8n_1x3_lf",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node5":{
            "core":"c8n_1x3_lf_shard1_replica_n2",
            "core_node_name":"core_node5",
            "base_url":"http://127.0.0.1:39657/v_/gn",
            "node_name":"127.0.0.1:39657_v_%2Fgn",
            "state":"recovering",
            "collection":"c8n_1x3_lf",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node6":{
            "core":"c8n_1x3_lf_shard1_replica_n4",
            "leader":"true",
            "core_node_name":"core_node6",
            "base_url":"http://127.0.0.1:40379/v_/gn",
            "node_name":"127.0.0.1:40379_v_%2Fgn",
            "state":"active",
            "collection":"c8n_1x3_lf",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"3",
    "tlogReplicas":"0"},
  "collection1":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{
      "shard1":{
        "range":"80000000-ffffffff",
        "state":"active",
        "replicas":{"core_node3":{
            "core":"collection1_shard1_replica_n1",
            "leader":"true",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:39657/v_/gn",
            "node_name":"127.0.0.1:39657_v_%2Fgn",
            "state":"active",
            "collection":"collection1",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}},
      "shard2":{
        "range":"0-7fffffff",
        "state":"active",
        "replicas":{
          "core_node4":{
            "core":"collection1_shard2_replica_n2",
            "leader":"true",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:40379/v_/gn",
            "node_name":"127.0.0.1:40379_v_%2Fgn",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node6":{
            "core":"collection1_shard2_replica_n5",
            "core_node_name":"core_node6",
            "base_url":"http://127.0.0.1:44599/v_/gn",
            "node_name":"127.0.0.1:44599_v_%2Fgn",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"}}
	at __randomizedtesting.SeedInfo.seed([5FF4BF6FD205CC18:D7A080B57CF9A1E0]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.ensureAllReplicasAreActive(AbstractFullDistribZkTestBase.java:2147)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:101)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:58)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:564)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1090)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1061)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:832)


FAILED:  org.apache.solr.cloud.LeaderFailureAfterFreshStartTest.test

Error Message:
java.lang.AssertionError: timeout waiting to see all nodes active

Stack Trace:
java.lang.AssertionError: timeout waiting to see all nodes active
	at __randomizedtesting.SeedInfo.seed([5FF4BF6FD205CC18:D7A080B57CF9A1E0]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.solr.cloud.LeaderFailureAfterFreshStartTest.waitTillNodesActive(LeaderFailureAfterFreshStartTest.java:234)
	at org.apache.solr.cloud.LeaderFailureAfterFreshStartTest.restartNodes(LeaderFailureAfterFreshStartTest.java:174)
	at org.apache.solr.cloud.LeaderFailureAfterFreshStartTest.test(LeaderFailureAfterFreshStartTest.java:149)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:564)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1090)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1061)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:832)


FAILED:  org.apache.solr.cloud.LeaderVoteWaitTimeoutTest.testMostInSyncReplicasCanWinElection

Error Message:
java.lang.AssertionError: Timeout waiting for 1x3 collection
Timeout waiting to see state for collection=collection1 :DocCollection(collection1/26)={
  "pullReplicas":"0",
  "replicationFactor":"3",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node2":{
          "core":"collection1_shard1_replica_n1",
          "core_node_name":"core_node2",
          "base_url":"https://127.0.0.1:34241/solr",
          "node_name":"127.0.0.1:34241_solr",
          "state":"recovering",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node4":{
          "core":"collection1_shard1_replica_n3",
          "core_node_name":"core_node4",
          "base_url":"https://127.0.0.1:39851/solr",
          "node_name":"127.0.0.1:39851_solr",
          "state":"active",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node6":{
          "core":"collection1_shard1_replica_n5",
          "leader":"true",
          "core_node_name":"core_node6",
          "base_url":"https://127.0.0.1:40607/solr",
          "node_name":"127.0.0.1:40607_solr",
          "state":"active",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"3",
  "tlogReplicas":"0"}
Live Nodes: [127.0.0.1:34241_solr, 127.0.0.1:39851_solr, 127.0.0.1:40607_solr, 127.0.0.1:41411_solr]
Last available state: DocCollection(collection1/26)={
  "pullReplicas":"0",
  "replicationFactor":"3",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node2":{
          "core":"collection1_shard1_replica_n1",
          "core_node_name":"core_node2",
          "base_url":"https://127.0.0.1:34241/solr",
          "node_name":"127.0.0.1:34241_solr",
          "state":"recovering",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node4":{
          "core":"collection1_shard1_replica_n3",
          "core_node_name":"core_node4",
          "base_url":"https://127.0.0.1:39851/solr",
          "node_name":"127.0.0.1:39851_solr",
          "state":"active",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node6":{
          "core":"collection1_shard1_replica_n5",
          "leader":"true",
          "core_node_name":"core_node6",
          "base_url":"https://127.0.0.1:40607/solr",
          "node_name":"127.0.0.1:40607_solr",
          "state":"active",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"3",
  "tlogReplicas":"0"}

Stack Trace:
java.lang.AssertionError: Timeout waiting for 1x3 collection
Timeout waiting to see state for collection=collection1 :DocCollection(collection1/26)={
  "pullReplicas":"0",
  "replicationFactor":"3",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node2":{
          "core":"collection1_shard1_replica_n1",
          "core_node_name":"core_node2",
          "base_url":"https://127.0.0.1:34241/solr",
          "node_name":"127.0.0.1:34241_solr",
          "state":"recovering",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node4":{
          "core":"collection1_shard1_replica_n3",
          "core_node_name":"core_node4",
          "base_url":"https://127.0.0.1:39851/solr",
          "node_name":"127.0.0.1:39851_solr",
          "state":"active",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node6":{
          "core":"collection1_shard1_replica_n5",
          "leader":"true",
          "core_node_name":"core_node6",
          "base_url":"https://127.0.0.1:40607/solr",
          "node_name":"127.0.0.1:40607_solr",
          "state":"active",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"3",
  "tlogReplicas":"0"}
Live Nodes: [127.0.0.1:34241_solr, 127.0.0.1:39851_solr, 127.0.0.1:40607_solr, 127.0.0.1:41411_solr]
Last available state: DocCollection(collection1/26)={
  "pullReplicas":"0",
  "replicationFactor":"3",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node2":{
          "core":"collection1_shard1_replica_n1",
          "core_node_name":"core_node2",
          "base_url":"https://127.0.0.1:34241/solr",
          "node_name":"127.0.0.1:34241_solr",
          "state":"recovering",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node4":{
          "core":"collection1_shard1_replica_n3",
          "core_node_name":"core_node4",
          "base_url":"https://127.0.0.1:39851/solr",
          "node_name":"127.0.0.1:39851_solr",
          "state":"active",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node6":{
          "core":"collection1_shard1_replica_n5",
          "leader":"true",
          "core_node_name":"core_node6",
          "base_url":"https://127.0.0.1:40607/solr",
     

[...truncated too long message...]

 "-Ptests.jvmargs=-XX:+UseCompressedOops -XX:+UseSerialGC" -Ptests.seed=5FF4BF6FD205CC18 -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=US-ASCII

  - org.apache.solr.cloud.LeaderVoteWaitTimeoutTest.testMostInSyncReplicasCanWinElection (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.LeaderVoteWaitTimeoutTest.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.LeaderVoteWaitTimeoutTest" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:+UseCompressedOops -XX:+UseSerialGC" -Ptests.seed=5FF4BF6FD205CC18 -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=US-ASCII

  - org.apache.solr.cloud.HttpPartitionTest.test (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.HttpPartitionTest.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.HttpPartitionTest" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:+UseCompressedOops -XX:+UseSerialGC" -Ptests.seed=5FF4BF6FD205CC18 -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=US-ASCII

  - org.apache.solr.cloud.SyncSliceTest.test (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.SyncSliceTest.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.SyncSliceTest" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:+UseCompressedOops -XX:+UseSerialGC" -Ptests.seed=5FF4BF6FD205CC18 -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=US-ASCII

  - org.apache.solr.cloud.TestCloudRecovery.corruptedLogTest (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.TestCloudRecovery.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.TestCloudRecovery" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:+UseCompressedOops -XX:+UseSerialGC" -Ptests.seed=5FF4BF6FD205CC18 -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=US-ASCII

  - org.apache.solr.cloud.MigrateRouteKeyTest.multipleShardMigrateTest (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.MigrateRouteKeyTest.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.MigrateRouteKeyTest" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:+UseCompressedOops -XX:+UseSerialGC" -Ptests.seed=5FF4BF6FD205CC18 -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=US-ASCII

  - org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.RecoveryAfterSoftCommitTest.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.RecoveryAfterSoftCommitTest" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:+UseCompressedOops -XX:+UseSerialGC" -Ptests.seed=5FF4BF6FD205CC18 -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=US-ASCII

  - org.apache.solr.cloud.ReplaceNodeTest.test (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.ReplaceNodeTest.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.ReplaceNodeTest" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:+UseCompressedOops -XX:+UseSerialGC" -Ptests.seed=5FF4BF6FD205CC18 -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=US-ASCII

  - org.apache.solr.cloud.TestLeaderElectionWithEmptyReplica.test (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.TestLeaderElectionWithEmptyReplica.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.TestLeaderElectionWithEmptyReplica" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:+UseCompressedOops -XX:+UseSerialGC" -Ptests.seed=5FF4BF6FD205CC18 -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=US-ASCII

  - org.apache.solr.cloud.DeleteReplicaTest.raceConditionOnDeleteAndRegisterReplica (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.DeleteReplicaTest.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.DeleteReplicaTest" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:+UseCompressedOops -XX:+UseSerialGC" -Ptests.seed=5FF4BF6FD205CC18 -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=US-ASCII

  - org.apache.solr.core.snapshots.TestSolrCloudSnapshots.testSnapshots (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.core.snapshots.TestSolrCloudSnapshots.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.core.snapshots.TestSolrCloudSnapshots" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:+UseCompressedOops -XX:+UseSerialGC" -Ptests.seed=5FF4BF6FD205CC18 -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=US-ASCII

  - org.apache.solr.cloud.MoveReplicaTest.test (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.MoveReplicaTest.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.MoveReplicaTest" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:+UseCompressedOops -XX:+UseSerialGC" -Ptests.seed=5FF4BF6FD205CC18 -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=US-ASCII

  - org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.ChaosMonkeySafeLeaderTest" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:+UseCompressedOops -XX:+UseSerialGC" -Ptests.seed=5FF4BF6FD205CC18 -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=US-ASCII

  - org.apache.solr.cloud.RecoveryZkTest.test (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.RecoveryZkTest.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.RecoveryZkTest" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:+UseCompressedOops -XX:+UseSerialGC" -Ptests.seed=5FF4BF6FD205CC18 -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=US-ASCII


Deprecated Gradle features were used in this build, making it incompatible with Gradle 7.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/6.6.1/userguide/command_line_interface.html#sec:command_line_warnings

BUILD SUCCESSFUL in 1h 7m 9s
854 actionable tasks: 854 executed
Build step 'Invoke Gradle script' changed build result to SUCCESS
Archiving artifacts
java.lang.InterruptedException: no matches found within 10000
	at hudson.FilePath$ValidateAntFileMask.hasMatch(FilePath.java:2875)
	at hudson.FilePath$ValidateAntFileMask.invoke(FilePath.java:2754)
	at hudson.FilePath$ValidateAntFileMask.invoke(FilePath.java:2735)
	at hudson.FilePath.act(FilePath.java:1076)
	at hudson.FilePath.act(FilePath.java:1059)
	at hudson.FilePath.validateAntFileMask(FilePath.java:2733)
	at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:270)
	at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:80)
	at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
	at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:804)
	at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:753)
	at hudson.model.Build$BuildExecution.post2(Build.java:177)
	at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:698)
	at hudson.model.Run.execute(Run.java:1919)
	at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
	at hudson.model.ResourceController.execute(ResourceController.java:97)
	at hudson.model.Executor.run(Executor.java:428)
No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/*_pid*.log". Configuration error?
[Java] Sleeping for 5 seconds due to JENKINS-32191...
[Java] Parsing console log (workspace: '/home/jenkins/workspace/Lucene-Solr-master-Linux')
[Java] Successfully parsed console log
[Java] -> found 0 issues (skipped 0 duplicates)
[Java] Skipping post processing
[Java] No filter has been set, publishing all 0 issues
[Java] Using reference build 'Lucene-Solr-master-Linux #28360' to compute new, fixed, and outstanding issues
[Java] Issues delta (vs. reference build): outstanding: 0, new: 0, fixed: 0
[Java] No quality gates have been set - skipping
[Java] Health report is disabled - skipping
[Java] Created analysis result for 0 issues (found 0 new issues, fixed 0 issues)
[Java] Attaching ResultAction with ID 'java' to run 'Lucene-Solr-master-Linux #28361'.
[Checks API] No suitable checks publisher found.
Recording test results
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
[Checks API] No suitable checks publisher found.
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)

[JENKINS] Lucene-Solr-master-Linux (64bit/jdk-11.0.6) - Build # 28360 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Linux/28360/
Java: 64bit/jdk-11.0.6 -XX:-UseCompressedOops -XX:+UseParallelGC

51 tests failed.
FAILED:  org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest.test

Error Message:
java.util.concurrent.TimeoutException: Cluster state still in flux after 30 retry intervals.

Stack Trace:
java.util.concurrent.TimeoutException: Cluster state still in flux after 30 retry intervals.
	at __randomizedtesting.SeedInfo.seed([E13C1A72561A0CDE:696825A8F8E66126]:0)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForThingsToLevelOut(AbstractFullDistribZkTestBase.java:1643)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForThingsToLevelOut(AbstractFullDistribZkTestBase.java:1613)
	at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest.test(ChaosMonkeyNothingIsSafeWithPullReplicasTest.java:253)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1090)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1061)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:834)


FAILED:  org.apache.solr.schema.TestPointFields.testFloatPointStats

Error Message:
java.lang.RuntimeException: XPath is invalid

Stack Trace:
java.lang.RuntimeException: XPath is invalid
	at __randomizedtesting.SeedInfo.seed([E13C1A72561A0CDE:5B366255EE0BCE3C]:0)
	at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:997)
	at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:960)
	at org.apache.solr.schema.TestPointFields.doTestPointStats(TestPointFields.java:2637)
	at org.apache.solr.schema.TestPointFields.testFloatPointStats(TestPointFields.java:1145)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:834)
Caused by: javax.xml.xpath.XPathExpressionException: javax.xml.transform.TransformerException: 无法将5.735999447388283E-4的格式设置为数字!
	at java.xml/com.sun.org.apache.xpath.internal.jaxp.XPathImpl.evaluate(XPathImpl.java:145)
	at org.apache.solr.util.BaseTestHarness.validateXPath(BaseTestHarness.java:92)
	at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:985)
	... 46 more
Caused by: javax.xml.transform.TransformerException: 无法将5.735999447388283E-4的格式设置为数字!
	at java.xml/com.sun.org.apache.xpath.internal.compiler.XPathParser.error(XPathParser.java:621)
	at java.xml/com.sun.org.apache.xpath.internal.compiler.XPathParser.Number(XPathParser.java:2045)
	at java.xml/com.sun.org.apache.xpath.internal.compiler.XPathParser.PrimaryExpr(XPathParser.java:1403)
	at java.xml/com.sun.org.apache.xpath.internal.compiler.XPathParser.FilterExpr(XPathParser.java:1311)
	at java.xml/com.sun.org.apache.xpath.internal.compiler.XPathParser.PathExpr(XPathParser.java:1244)
	at java.xml/com.sun.org.apache.xpath.internal.compiler.XPathParser.UnionExpr(XPathParser.java:1202)
	at java.xml/com.sun.org.apache.xpath.internal.compiler.XPathParser.UnaryExpr(XPathParser.java:1108)
	at java.xml/com.sun.org.apache.xpath.internal.compiler.XPathParser.MultiplicativeExpr(XPathParser.java:1029)
	at java.xml/com.sun.org.apache.xpath.internal.compiler.XPathParser.AdditiveExpr(XPathParser.java:971)
	at java.xml/com.sun.org.apache.xpath.internal.compiler.XPathParser.RelationalExpr(XPathParser.java:896)
	at java.xml/com.sun.org.apache.xpath.internal.compiler.XPathParser.RelationalExpr(XPathParser.java:937)
	at java.xml/com.sun.org.apache.xpath.internal.compiler.XPathParser.EqualityExpr(XPathParser.java:836)
	at java.xml/com.sun.org.apache.xpath.internal.compiler.XPathParser.AndExpr(XPathParser.java:800)
	at java.xml/com.sun.org.apache.xpath.internal.compiler.XPathParser.OrExpr(XPathParser.java:773)
	at java.xml/com.sun.org.apache.xpath.internal.compiler.XPathParser.Expr(XPathParser.java:756)
	at java.xml/com.sun.org.apache.xpath.internal.compiler.XPathParser.PredicateExpr(XPathParser.java:1921)
	at java.xml/com.sun.org.apache.xpath.internal.compiler.XPathParser.Predicate(XPathParser.java:1902)
	at java.xml/com.sun.org.apache.xpath.internal.compiler.XPathParser.Step(XPathParser.java:1692)
	at java.xml/com.sun.org.apache.xpath.internal.compiler.XPathParser.RelativeLocationPath(XPathParser.java:1601)
	at java.xml/com.sun.org.apache.xpath.internal.compiler.XPathParser.LocationPath(XPathParser.java:1563)
	at java.xml/com.sun.org.apache.xpath.internal.compiler.XPathParser.PathExpr(XPathParser.java:1283)
	at java.xml/com.sun.org.apache.xpath.internal.compiler.XPathParser.UnionExpr(XPathParser.java:1202)
	at java.xml/com.sun.org.apache.xpath.internal.compiler.XPathParser.UnaryExpr(XPathParser.java:1108)
	at java.xml/com.sun.org.apache.xpath.internal.compiler.XPathParser.MultiplicativeExpr(XPathParser.java:1029)
	at java.xml/com.sun.org.apache.xpath.internal.compiler.XPathParser.AdditiveExpr(XPathParser.java:971)
	at java.xml/com.sun.org.apache.xpath.internal.compiler.XPathParser.RelationalExpr(XPathParser.java:896)
	at java.xml/com.sun.org.apache.xpath.internal.compiler.XPathParser.EqualityExpr(XPathParser.java:836)
	at java.xml/com.sun.org.apache.xpath.internal.compiler.XPathParser.AndExpr(XPathParser.java:800)
	at java.xml/com.sun.org.apache.xpath.internal.compiler.XPathParser.OrExpr(XPathParser.java:773)
	at java.xml/com.sun.org.apache.xpath.internal.compiler.XPathParser.Expr(XPathParser.java:756)
	at java.xml/com.sun.org.apache.xpath.internal.compiler.XPathParser.initXPath(XPathParser.java:132)
	at java.xml/com.sun.org.apache.xpath.internal.XPath.<init>(XPath.java:178)
	at java.xml/com.sun.org.apache.xpath.internal.XPath.<init>(XPath.java:268)
	at java.xml/com.sun.org.apache.xpath.internal.jaxp.XPathImpl.eval(XPathImpl.java:116)
	at java.xml/com.sun.org.apache.xpath.internal.jaxp.XPathImpl.evaluate(XPathImpl.java:131)
	... 48 more


FAILED:  org.apache.solr.cloud.MoveReplicaTest.test

Error Message:
java.lang.AssertionError

Stack Trace:
java.lang.AssertionError
	at __randomizedtesting.SeedInfo.seed([E13C1A72561A0CDE:696825A8F8E66126]:0)
	at org.junit.Assert.fail(Assert.java:87)
	at org.junit.Assert.assertTrue(Assert.java:42)
	at org.junit.Assert.assertTrue(Assert.java:53)
	at org.apache.solr.cloud.MoveReplicaTest.test(MoveReplicaTest.java:150)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:834)


FAILED:  org.apache.solr.cloud.BasicDistributedZk2Test.test

Error Message:
java.lang.AssertionError: There are still nodes recovering - waited for 330SECONDS

Stack Trace:
java.lang.AssertionError: There are still nodes recovering - waited for 330SECONDS
	at __randomizedtesting.SeedInfo.seed([E13C1A72561A0CDE:696825A8F8E66126]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:197)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:144)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:139)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:1051)
	at org.apache.solr.cloud.BasicDistributedZk2Test.brindDownShardIndexSomeDocsAndRecover(BasicDistributedZk2Test.java:362)
	at org.apache.solr.cloud.BasicDistributedZk2Test.test(BasicDistributedZk2Test.java:103)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1090)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1061)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:834)


FAILED:  org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest.test

Error Message:
java.lang.AssertionError: There are still nodes recovering - waited for 180SECONDS

Stack Trace:
java.lang.AssertionError: There are still nodes recovering - waited for 180SECONDS
	at __randomizedtesting.SeedInfo.seed([E13C1A72561A0CDE:696825A8F8E66126]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:197)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:150)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:1063)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForThingsToLevelOut(AbstractFullDistribZkTestBase.java:1621)
	at org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest.test(ChaosMonkeySafeLeaderWithPullReplicasTest.java:198)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1090)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1061)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:834)


FAILED:  org.apache.solr.cloud.DeleteReplicaTest.raceConditionOnDeleteAndRegisterReplica

Error Message:
java.lang.RuntimeException: Failed while waiting for active collection
Timeout waiting to see state for collection=raceDeleteReplicaCollection :DocCollection(raceDeleteReplicaCollection/17)={
  "pullReplicas":"0",
  "replicationFactor":"2",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node4":{
          "core":"raceDeleteReplicaCollection_shard1_replica_n2",
          "core_node_name":"core_node4",
          "base_url":"http://127.0.0.1:42823/solr",
          "node_name":"127.0.0.1:42823_solr",
          "state":"recovering",
          "collection":"raceDeleteReplicaCollection",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node6":{
          "core":"raceDeleteReplicaCollection_shard1_replica_n5",
          "leader":"true",
          "core_node_name":"core_node6",
          "base_url":"http://127.0.0.1:39555/solr",
          "node_name":"127.0.0.1:39555_solr",
          "state":"active",
          "collection":"raceDeleteReplicaCollection",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"2",
  "tlogReplicas":"0"}
Live Nodes: [127.0.0.1:34917_solr, 127.0.0.1:38517_solr, 127.0.0.1:39555_solr, 127.0.0.1:42823_solr]
Last available state: DocCollection(raceDeleteReplicaCollection/17)={
  "pullReplicas":"0",
  "replicationFactor":"2",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node4":{
          "core":"raceDeleteReplicaCollection_shard1_replica_n2",
          "core_node_name":"core_node4",
          "base_url":"http://127.0.0.1:42823/solr",
          "node_name":"127.0.0.1:42823_solr",
          "state":"recovering",
          "collection":"raceDeleteReplicaCollection",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node6":{
          "core":"raceDeleteReplicaCollection_shard1_replica_n5",
          "leader":"true",
          "core_node_name":"core_node6",
          "base_url":"http://127.0.0.1:39555/solr",
          "node_name":"127.0.0.1:39555_solr",
          "state":"active",
          "collection":"raceDeleteReplicaCollection",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"2",
  "tlogReplicas":"0"}

Stack Trace:
java.lang.RuntimeException: Failed while waiting for active collection
Timeout waiting to see state for collection=raceDeleteReplicaCollection :DocCollection(raceDeleteReplicaCollection/17)={
  "pullReplicas":"0",
  "replicationFactor":"2",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node4":{
          "core":"raceDeleteReplicaCollection_shard1_replica_n2",
          "core_node_name":"core_node4",
          "base_url":"http://127.0.0.1:42823/solr",
          "node_name":"127.0.0.1:42823_solr",
          "state":"recovering",
          "collection":"raceDeleteReplicaCollection",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node6":{
          "core":"raceDeleteReplicaCollection_shard1_replica_n5",
          "leader":"true",
          "core_node_name":"core_node6",
          "base_url":"http://127.0.0.1:39555/solr",
          "node_name":"127.0.0.1:39555_solr",
          "state":"active",
          "collection":"raceDeleteReplicaCollection",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"2",
  "tlogReplicas":"0"}
Live Nodes: [127.0.0.1:34917_solr, 127.0.0.1:38517_solr, 127.0.0.1:39555_solr, 127.0.0.1:42823_solr]
Last available state: DocCollection(raceDeleteReplicaCollection/17)={
  "pullReplicas":"0",
  "replicationFactor":"2",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node4":{
          "core":"raceDeleteReplicaCollection_shard1_replica_n2",
          "core_node_name":"core_node4",
          "base_url":"http://127.0.0.1:42823/solr",
          "node_name":"127.0.0.1:42823_solr",
          "state":"recovering",
          "collection":"raceDeleteReplicaCollection",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node6":{
          "core":"raceDeleteReplicaCollection_shard1_replica_n5",
          "leader":"true",
          "core_node_name":"core_node6",
          "base_url":"http://127.0.0.1:39555/solr",
          "node_name":"127.0.0.1:39555_solr",
          "state":"active",
          "collection":"raceDeleteReplicaCollection",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"2",
  "tlogReplicas":"0"}
	at __randomizedtesting.SeedInfo.seed([E13C1A72561A0CDE:8B2A7BA23EE84614]:0)
	at org.apache.solr.cloud.MiniSolrCloudCluster.waitForActiveCollection(MiniSolrCloudCluster.java:774)
	at org.apache.solr.cloud.MiniSolrCloudCluster.waitForActiveCollection(MiniSolrCloudCluster.java:780)
	at org.apache.solr.cloud.DeleteReplicaTest.raceConditionOnDeleteAndRegisterReplica(DeleteReplicaTest.java:385)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:834)


FAILED:  org.apache.solr.cloud.HttpPartitionTest.test

Error Message:
java.lang.AssertionError: Didn't see all replicas for shard shard1 in collDoRecoveryOnRestart come up within 130000 ms! ClusterState: {
  "control_collection":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{"core_node2":{
            "core":"control_collection_shard1_replica_n1",
            "leader":"true",
            "core_node_name":"core_node2",
            "base_url":"http://127.0.0.1:44487",
            "node_name":"127.0.0.1:44487_",
            "state":"active",
            "collection":"control_collection",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"},
  "collDoRecoveryOnRestart":{
    "pullReplicas":"0",
    "replicationFactor":"2",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"collDoRecoveryOnRestart_shard1_replica_n1",
            "leader":"true",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:44487",
            "node_name":"127.0.0.1:44487_",
            "state":"active",
            "collection":"collDoRecoveryOnRestart",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node4":{
            "core":"collDoRecoveryOnRestart_shard1_replica_n2",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:45537",
            "node_name":"127.0.0.1:45537_",
            "state":"recovering",
            "collection":"collDoRecoveryOnRestart",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"2",
    "tlogReplicas":"0"},
  "collection1":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{
      "shard1":{
        "range":"80000000-ffffffff",
        "state":"active",
        "replicas":{"core_node4":{
            "core":"collection1_shard1_replica_n2",
            "leader":"true",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:45537",
            "node_name":"127.0.0.1:45537_",
            "state":"active",
            "collection":"collection1",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}},
      "shard2":{
        "range":"0-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"collection1_shard2_replica_n1",
            "leader":"true",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:43247",
            "node_name":"127.0.0.1:43247_",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node6":{
            "core":"collection1_shard2_replica_n5",
            "core_node_name":"core_node6",
            "base_url":"http://127.0.0.1:41241",
            "node_name":"127.0.0.1:41241_",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"}}

Stack Trace:
java.lang.AssertionError: Didn't see all replicas for shard shard1 in collDoRecoveryOnRestart come up within 130000 ms! ClusterState: {
  "control_collection":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{"core_node2":{
            "core":"control_collection_shard1_replica_n1",
            "leader":"true",
            "core_node_name":"core_node2",
            "base_url":"http://127.0.0.1:44487",
            "node_name":"127.0.0.1:44487_",
            "state":"active",
            "collection":"control_collection",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"},
  "collDoRecoveryOnRestart":{
    "pullReplicas":"0",
    "replicationFactor":"2",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"collDoRecoveryOnRestart_shard1_replica_n1",
            "leader":"true",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:44487",
            "node_name":"127.0.0.1:44487_",
            "state":"active",
            "collection":"collDoRecoveryOnRestart",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node4":{
            "core":"collDoRecoveryOnRestart_shard1_replica_n2",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:45537",
            "node_name":"127.0.0.1:45537_",
            "state":"recovering",
            "collection":"collDoRecoveryOnRestart",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"2",
    "tlogReplicas":"0"},
  "collection1":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{
      "shard1":{
        "range":"80000000-ffffffff",
        "state":"active",
        "replicas":{"core_node4":{
            "core":"collection1_shard1_replica_n2",
            "leader":"true",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:45537",
            "node_name":"127.0.0.1:45537_",
            "state":"active",
            "collection":"collection1",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}},
      "shard2":{
        "range":"0-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"collection1_shard2_replica_n1",
            "leader":"true",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:43247",
            "node_name":"127.0.0.1:43247_",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node6":{
            "core":"collection1_shard2_replica_n5",
            "core_node_name":"core_node6",
            "base_url":"http://127.0.0.1:41241",
            "node_name":"127.0.0.1:41241_",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"}}
	at __randomizedtesting.SeedInfo.seed([E13C1A72561A0CDE:696825A8F8E66126]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.ensureAllReplicasAreActive(AbstractFullDistribZkTestBase.java:2147)
	at org.apache.solr.cloud.HttpPartitionTest.testDoRecoveryOnRestart(HttpPartitionTest.java:200)
	at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:132)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1090)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1061)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:834)


FAILED:  org.apache.solr.cloud.HttpPartitionWithTlogReplicasTest.test

Error Message:
java.lang.AssertionError: Didn't see all replicas for shard shard1 in collDoRecoveryOnRestart come up within 130000 ms! ClusterState: {
  "control_collection":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{"core_node2":{
            "core":"control_collection_shard1_replica_n1",
            "leader":"true",
            "core_node_name":"core_node2",
            "base_url":"http://127.0.0.1:34425/qs/s",
            "node_name":"127.0.0.1:34425_qs%2Fs",
            "state":"active",
            "collection":"control_collection",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"},
  "collDoRecoveryOnRestart":{
    "pullReplicas":"0",
    "replicationFactor":"0",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"collDoRecoveryOnRestart_shard1_replica_t1",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:38645/qs/s",
            "node_name":"127.0.0.1:38645_qs%2Fs",
            "state":"recovering",
            "collection":"collDoRecoveryOnRestart",
            "shard":"shard1",
            "type":"TLOG",
            "force_set_state":"false"},
          "core_node4":{
            "core":"collDoRecoveryOnRestart_shard1_replica_t2",
            "leader":"true",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:34425/qs/s",
            "node_name":"127.0.0.1:34425_qs%2Fs",
            "state":"active",
            "collection":"collDoRecoveryOnRestart",
            "shard":"shard1",
            "type":"TLOG",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"0",
    "tlogReplicas":"2"},
  "collection1":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{
      "shard1":{
        "range":"80000000-ffffffff",
        "state":"active",
        "replicas":{"core_node4":{
            "core":"collection1_shard1_replica_t2",
            "leader":"true",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:33111/qs/s",
            "node_name":"127.0.0.1:33111_qs%2Fs",
            "state":"active",
            "collection":"collection1",
            "shard":"shard1",
            "type":"TLOG",
            "force_set_state":"false"}}},
      "shard2":{
        "range":"0-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"collection1_shard2_replica_t1",
            "leader":"true",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:34375/qs/s",
            "node_name":"127.0.0.1:34375_qs%2Fs",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"TLOG",
            "force_set_state":"false"},
          "core_node6":{
            "core":"collection1_shard2_replica_t5",
            "core_node_name":"core_node6",
            "base_url":"http://127.0.0.1:38645/qs/s",
            "node_name":"127.0.0.1:38645_qs%2Fs",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"TLOG",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"}}

Stack Trace:
java.lang.AssertionError: Didn't see all replicas for shard shard1 in collDoRecoveryOnRestart come up within 130000 ms! ClusterState: {
  "control_collection":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{"core_node2":{
            "core":"control_collection_shard1_replica_n1",
            "leader":"true",
            "core_node_name":"core_node2",
            "base_url":"http://127.0.0.1:34425/qs/s",
            "node_name":"127.0.0.1:34425_qs%2Fs",
            "state":"active",
            "collection":"control_collection",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"},
  "collDoRecoveryOnRestart":{
    "pullReplicas":"0",
    "replicationFactor":"0",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"collDoRecoveryOnRestart_shard1_replica_t1",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:38645/qs/s",
            "node_name":"127.0.0.1:38645_qs%2Fs",
            "state":"recovering",
            "collection":"collDoRecoveryOnRestart",
            "shard":"shard1",
            "type":"TLOG",
            "force_set_state":"false"},
          "core_node4":{
            "core":"collDoRecoveryOnRestart_shard1_replica_t2",
            "leader":"true",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:34425/qs/s",
            "node_name":"127.0.0.1:34425_qs%2Fs",
            "state":"active",
            "collection":"collDoRecoveryOnRestart",
            "shard":"shard1",
            "type":"TLOG",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"0",
    "tlogReplicas":"2"},
  "collection1":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{
      "shard1":{
        "range":"80000000-ffffffff",
        "state":"active",
        "replicas":{"core_node4":{
            "core":"collection1_shard1_replica_t2",
            "leader":"true",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:33111/qs/s",
            "node_name":"127.0.0.1:33111_qs%2Fs",
            "state":"active",
            "collection":"collection1",
            "shard":"shard1",
            "type":"TLOG",
            "force_set_state":"false"}}},
      "shard2":{
        "range":"0-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"collection1_shard2_replica_t1",
            "leader":"true",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:34375/qs/s",
            "node_name":"127.0.0.1:34375_qs%2Fs",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"TLOG",
            "force_set_state":"false"},
          "core_node6":{
            "core":"collection1_shard2_replica_t5",
            "core_node_name":"core_node6",
            "base_url":"http://127.0.0.1:38645/qs/s",
            "node_name":"127.0.0.1:38645_qs%2Fs",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"TLOG",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"}}
	at __randomizedtesting.SeedInfo.seed([E13C1A72561A0CDE:696825A8F8E66126]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.ensureAllReplicasAreActive(AbstractFullDistribZkTestBase.java:2147)
	at org.apache.solr.cloud.HttpPartitionTest.testDoRecoveryOnRestart(HttpPartitionTest.java:200)
	at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:132)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1090)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1061)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:834)


FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test

Error Message:
java.lang.AssertionError: Didn't see all replicas for shard shard1 in c8n_1x3_lf come up within 90000 ms! ClusterState: {
  "control_collection":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{"core_node2":{
            "core":"control_collection_shard1_replica_n1",
            "leader":"true",
            "core_node_name":"core_node2",
            "base_url":"http://127.0.0.1:40529",
            "node_name":"127.0.0.1:40529_",
            "state":"active",
            "collection":"control_collection",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"},
  "c8n_1x3_lf":{
    "pullReplicas":"0",
    "replicationFactor":"3",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"c8n_1x3_lf_shard1_replica_n1",
            "leader":"true",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:37277",
            "node_name":"127.0.0.1:37277_",
            "state":"active",
            "collection":"c8n_1x3_lf",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node5":{
            "core":"c8n_1x3_lf_shard1_replica_n2",
            "core_node_name":"core_node5",
            "base_url":"http://127.0.0.1:34377",
            "node_name":"127.0.0.1:34377_",
            "state":"recovering",
            "collection":"c8n_1x3_lf",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node6":{
            "core":"c8n_1x3_lf_shard1_replica_n4",
            "core_node_name":"core_node6",
            "base_url":"http://127.0.0.1:40529",
            "node_name":"127.0.0.1:40529_",
            "state":"recovering",
            "collection":"c8n_1x3_lf",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"3",
    "tlogReplicas":"0"},
  "collection1":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{
      "shard1":{
        "range":"80000000-ffffffff",
        "state":"active",
        "replicas":{"core_node4":{
            "core":"collection1_shard1_replica_n2",
            "leader":"true",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:32839",
            "node_name":"127.0.0.1:32839_",
            "state":"active",
            "collection":"collection1",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}},
      "shard2":{
        "range":"0-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"collection1_shard2_replica_n1",
            "leader":"true",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:37277",
            "node_name":"127.0.0.1:37277_",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node6":{
            "core":"collection1_shard2_replica_n5",
            "core_node_name":"core_node6",
            "base_url":"http://127.0.0.1:34377",
            "node_name":"127.0.0.1:34377_",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"}}

Stack Trace:
java.lang.AssertionError: Didn't see all replicas for shard shard1 in c8n_1x3_lf come up within 90000 ms! ClusterState: {
  "control_collection":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{"core_node2":{
            "core":"control_collection_shard1_replica_n1",
            "leader":"true",
            "core_node_name":"core_node2",
            "base_url":"http://127.0.0.1:40529",
            "node_name":"127.0.0.1:40529_",
            "state":"active",
            "collection":"control_collection",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"},
  "c8n_1x3_lf":{
    "pullReplicas":"0",
    "replicationFactor":"3",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"c8n_1x3_lf_shard1_replica_n1",
            "leader":"true",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:37277",
            "node_name":"127.0.0.1:37277_",
            "state":"active",
            "collection":"c8n_1x3_lf",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node5":{
            "core":"c8n_1x3_lf_shard1_replica_n2",
            "core_node_name":"core_node5",
            "base_url":"http://127.0.0.1:34377",
            "node_name":"127.0.0.1:34377_",
            "state":"recovering",
            "collection":"c8n_1x3_lf",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node6":{
            "core":"c8n_1x3_lf_shard1_replica_n4",
            "core_node_name":"core_node6",
            "base_url":"http://127.0.0.1:40529",
            "node_name":"127.0.0.1:40529_",
            "state":"recovering",
            "collection":"c8n_1x3_lf",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"3",
    "tlogReplicas":"0"},
  "collection1":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{
      "shard1":{
        "range":"80000000-ffffffff",
        "state":"active",
        "replicas":{"core_node4":{
            "core":"collection1_shard1_replica_n2",
            "leader":"true",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:32839",
            "node_name":"127.0.0.1:32839_",
            "state":"active",
            "collection":"collection1",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}},
      "shard2":{
        "range":"0-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"collection1_shard2_replica_n1",
            "leader":"true",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:37277",
            "node_name":"127.0.0.1:37277_",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node6":{
            "core":"collection1_shard2_replica_n5",
            "core_node_name":"core_node6",
            "base_url":"http://127.0.0.1:34377",
            "node_name":"127.0.0.1:34377_",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"}}
	at __randomizedtesting.SeedInfo.seed([E13C1A72561A0CDE:696825A8F8E66126]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.ensureAllReplicasAreActive(AbstractFullDistribZkTestBase.java:2147)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:101)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:58)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1090)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1061)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:834)


FAILED:  org.apache.solr.cloud.LeaderFailureAfterFreshStartTest.test

Error Message:
java.lang.AssertionError: timeout waiting to see all nodes active

Stack Trace:
java.lang.AssertionError: timeout waiting to see all nodes active
	at __randomizedtesting.SeedInfo.seed([E13C1A72561A0CDE:696825A8F8E66126]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.solr.cloud.LeaderFailureAfterFreshStartTest.waitTillNodesActive(LeaderFailureAfterFreshStartTest.java:234)
	at org.apache.solr.cloud.LeaderFailureAfterFreshStartTest.restartNodes(LeaderFailureAfterFreshStartTest.java:174)
	at org.apache.solr.cloud.LeaderFailureAfterFreshStartTest.test(LeaderFailureAfterFreshStartTest.java:149)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1090)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1061)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:834)


FAILED:  org.apache.solr.cloud.LeaderVoteWaitTimeoutTest.testMostInSyncReplicasCanWinElection

Error Message:
java.lang.AssertionError: Timeout waiting for 1x3 collection
Timeout waiting to see state for collection=collection1 :DocCollection(collection1/26)={
  "pullReplicas":"0",
  "replicationFactor":"3",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node2":{
          "core":"collection1_shard1_replica_n1",
          "core_node_name":"core_node2",
          "base_url":"http://127.0.0.1:39919/solr",
          "node_name":"127.0.0.1:39919_solr",
          "state":"recovering",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node4":{
          "core":"collection1_shard1_replica_n3",
          "core_node_name":"core_node4",
          "base_url":"http://127.0.0.1:36019/solr",
          "node_name":"127.0.0.1:36019_solr",
          "state":"active",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node6":{
          "core":"collection1_shard1_replica_n5",
          "leader":"true",
          "core_node_name":"core_node6",
          "base_url":"http://127.0.0.1:36821/solr",
          "node_name":"127.0.0.1:36821_solr",
          "state":"active",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"3",
  "tlogReplicas":"0"}
Live Nodes: [127.0.0.1:36019_solr, 127.0.0.1:36821_solr, 127.0.0.1:39919_solr, 127.0.0.1:46725_solr]
Last available state: DocCollection(collection1/26)={
  "pullReplicas":"0",
  "replicationFactor":"3",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node2":{
          "core":"collection1_shard1_replica_n1",
          "core_node_name":"core_node2",
          "base_url":"http://127.0.0.1:39919/solr",
          "node_name":"127.0.0.1:39919_solr",
          "state":"recovering",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node4":{
          "core":"collection1_shard1_replica_n3",
          "core_node_name":"core_node4",
          "base_url":"http://127.0.0.1:36019/solr",
          "node_name":"127.0.0.1:36019_solr",
          "state":"active",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node6":{
          "core":"collection1_shard1_replica_n5",
          "leader":"true",
          "core_node_name":"core_node6",
          "base_url":"http://127.0.0.1:36821/solr",
          "node_name":"127.0.0.1:36821_solr",
          "state":"active",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"3",
  "tlogReplicas":"0"}

Stack Trace:
java.lang.AssertionError: Timeout waiting for 1x3 collection
Timeout waiting to see state for collection=collection1 :DocCollection(collection1/26)={
  "pullReplicas":"0",
  "replicationFactor":"3",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node2":{
          "core":"collection1_shard1_replica_n1",
          "core_node_name":"core_node2",
          "base_url":"http://127.0.0.1:39919/solr",
          "node_name":"127.0.0.1:39919_solr",
          "state":"recovering",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node4":{
          "core":"collection1_shard1_replica_n3",
          "core_node_name":"core_node4",
          "base_url":"http://127.0.0.1:36019/solr",
          "node_name":"127.0.0.1:36019_solr",
          "state":"active",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node6":{
          "core":"collection1_shard1_replica_n5",
          "leader":"true",
          "core_node_name":"core_node6",
          "base_url":"http://127.0.0.1:36821/solr",
          "node_name":"127.0.0.1:36821_solr",
          "state":"active",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"3",
  "tlogReplicas":"0"}
Live Nodes: [127.0.0.1:36019_solr, 127.0.0.1:36821_solr, 127.0.0.1:39919_solr, 127.0.0.1:46725_solr]
Last available state: DocCollection(collection1/26)={
  "pullReplicas":"0",
  "replicationFactor":"3",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node2":{
          "core":"collection1_shard1_replica_n1",
          "core_node_name":"core_node2",
          "base_url":"http://127.0.0.1:39919/solr",
          "node_name":"127.0.0.1:39919_solr",
          "state":"recovering",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node4":{
          "core":"collection1_shard1_replica_n3",
          "core_node_name":"core_node4",
          "base_url":"http://127.0.0.1:36019/solr",
          "node_name":"127.0.0.1:36019_solr",
          "state":"active",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node6":{
          "core":"collection1_shard1_replica_n5",
          "leader":"true",
          "core_node_name":"core_node6",
          "base_url":"http://127.0.0.1:36821/solr",
          "node_name":"127.0.0.1:36821_solr",
          "state":"active",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"3",
  "tlogReplicas":"0"}
	at __randomizedtesting.SeedInfo.seed([E13C1A72561A0CDE:492006C8945A38F4]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.solr.cloud.SolrCloudTestCase.waitForState(SolrCloudTestCase.java:326)
	at org.apache.solr.cloud.SolrCloudTestCase.waitForState(SolrCloudTestCase.java:303)
	at org.apache.solr.cloud.LeaderVoteWaitTimeoutTest.testMostInSyncReplicasCanWinElection(LeaderVoteWaitTimeoutTest.java:253)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.S

[...truncated too long message...]

multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=ISO-8859-1

  - org.apache.solr.cloud.LeaderFailureAfterFreshStartTest.test (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.LeaderFailureAfterFreshStartTest.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.LeaderFailureAfterFreshStartTest" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseParallelGC" -Ptests.seed=E13C1A72561A0CDE -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=ISO-8859-1

  - org.apache.solr.cloud.UnloadDistributedZkTest.test (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.UnloadDistributedZkTest.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.UnloadDistributedZkTest" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseParallelGC" -Ptests.seed=E13C1A72561A0CDE -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=ISO-8859-1

  - org.apache.solr.cloud.HttpPartitionTest.test (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.HttpPartitionTest.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.HttpPartitionTest" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseParallelGC" -Ptests.seed=E13C1A72561A0CDE -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=ISO-8859-1

  - org.apache.solr.cloud.LeaderVoteWaitTimeoutTest.testMostInSyncReplicasCanWinElection (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.LeaderVoteWaitTimeoutTest.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.LeaderVoteWaitTimeoutTest" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseParallelGC" -Ptests.seed=E13C1A72561A0CDE -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=ISO-8859-1

  - org.apache.solr.cloud.SyncSliceTest.test (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.SyncSliceTest.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.SyncSliceTest" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseParallelGC" -Ptests.seed=E13C1A72561A0CDE -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=ISO-8859-1

  - org.apache.solr.cloud.MigrateRouteKeyTest.multipleShardMigrateTest (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.MigrateRouteKeyTest.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.MigrateRouteKeyTest" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseParallelGC" -Ptests.seed=E13C1A72561A0CDE -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=ISO-8859-1

  - org.apache.solr.cloud.TestCloudRecovery.corruptedLogTest (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.TestCloudRecovery.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.TestCloudRecovery" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseParallelGC" -Ptests.seed=E13C1A72561A0CDE -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=ISO-8859-1

  - org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.RecoveryAfterSoftCommitTest.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.RecoveryAfterSoftCommitTest" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseParallelGC" -Ptests.seed=E13C1A72561A0CDE -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=ISO-8859-1

  - org.apache.solr.cloud.DeleteReplicaTest.raceConditionOnDeleteAndRegisterReplica (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.DeleteReplicaTest.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.DeleteReplicaTest" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseParallelGC" -Ptests.seed=E13C1A72561A0CDE -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=ISO-8859-1

  - org.apache.solr.cloud.TestLeaderElectionWithEmptyReplica.test (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.TestLeaderElectionWithEmptyReplica.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.TestLeaderElectionWithEmptyReplica" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseParallelGC" -Ptests.seed=E13C1A72561A0CDE -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=ISO-8859-1

  - org.apache.solr.cloud.MoveReplicaTest.test (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.MoveReplicaTest.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.MoveReplicaTest" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseParallelGC" -Ptests.seed=E13C1A72561A0CDE -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=ISO-8859-1

  - org.apache.solr.core.snapshots.TestSolrCloudSnapshots.testSnapshots (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.core.snapshots.TestSolrCloudSnapshots.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.core.snapshots.TestSolrCloudSnapshots" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseParallelGC" -Ptests.seed=E13C1A72561A0CDE -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=ISO-8859-1

  - org.apache.solr.cloud.RecoveryZkTest.test (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.RecoveryZkTest.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.RecoveryZkTest" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseParallelGC" -Ptests.seed=E13C1A72561A0CDE -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=ISO-8859-1


Deprecated Gradle features were used in this build, making it incompatible with Gradle 7.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/6.6.1/userguide/command_line_interface.html#sec:command_line_warnings

BUILD SUCCESSFUL in 55m 5s
853 actionable tasks: 853 executed
Build step 'Invoke Gradle script' changed build result to SUCCESS
Archiving artifacts
java.lang.InterruptedException: no matches found within 10000
	at hudson.FilePath$ValidateAntFileMask.hasMatch(FilePath.java:2875)
	at hudson.FilePath$ValidateAntFileMask.invoke(FilePath.java:2754)
	at hudson.FilePath$ValidateAntFileMask.invoke(FilePath.java:2735)
	at hudson.FilePath.act(FilePath.java:1076)
	at hudson.FilePath.act(FilePath.java:1059)
	at hudson.FilePath.validateAntFileMask(FilePath.java:2733)
	at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:270)
	at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:80)
	at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
	at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:804)
	at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:753)
	at hudson.model.Build$BuildExecution.post2(Build.java:177)
	at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:698)
	at hudson.model.Run.execute(Run.java:1919)
	at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
	at hudson.model.ResourceController.execute(ResourceController.java:97)
	at hudson.model.Executor.run(Executor.java:428)
No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/*_pid*.log". Configuration error?
[Java] Sleeping for 5 seconds due to JENKINS-32191...
[Java] Parsing console log (workspace: '/home/jenkins/workspace/Lucene-Solr-master-Linux')
[Java] Successfully parsed console log
[Java] -> found 0 issues (skipped 0 duplicates)
[Java] Skipping post processing
[Java] No filter has been set, publishing all 0 issues
[Java] Using reference build 'Lucene-Solr-master-Linux #28359' to compute new, fixed, and outstanding issues
[Java] Issues delta (vs. reference build): outstanding: 0, new: 0, fixed: 0
[Java] No quality gates have been set - skipping
[Java] Health report is disabled - skipping
[Java] Created analysis result for 0 issues (found 0 new issues, fixed 0 issues)
[Java] Attaching ResultAction with ID 'java' to run 'Lucene-Solr-master-Linux #28360'.
[Checks API] No suitable checks publisher found.
Recording test results
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
[Checks API] No suitable checks publisher found.
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)

[JENKINS] Lucene-Solr-master-Linux (64bit/jdk-12.0.2) - Build # 28359 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Linux/28359/
Java: 64bit/jdk-12.0.2 -XX:-UseCompressedOops -XX:+UseG1GC

51 tests failed.
FAILED:  org.apache.solr.cloud.MoveReplicaTest.test

Error Message:
java.lang.AssertionError: replica never fully recovered

Stack Trace:
java.lang.AssertionError: replica never fully recovered
	at __randomizedtesting.SeedInfo.seed([766B2CE6E90FD7C6:FE3F133C47F3BA3E]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.junit.Assert.assertTrue(Assert.java:42)
	at org.apache.solr.cloud.MoveReplicaTest.test(MoveReplicaTest.java:189)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:567)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:835)


FAILED:  org.apache.solr.cloud.AssignBackwardCompatibilityTest.test

Error Message:
java.lang.RuntimeException: Failed while waiting for active collection
Timeout waiting to see state for collection=collection1 :DocCollection(collection1/39)={
  "pullReplicas":"0",
  "replicationFactor":"4",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node8":{
          "core":"collection1_shard1_replica_n7",
          "leader":"true",
          "core_node_name":"core_node8",
          "base_url":"https://127.0.0.1:34937/solr",
          "node_name":"127.0.0.1:34937_solr",
          "state":"active",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node10":{
          "core":"collection1_shard1_replica_n9",
          "core_node_name":"core_node10",
          "base_url":"https://127.0.0.1:36053/solr",
          "node_name":"127.0.0.1:36053_solr",
          "state":"recovering",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"4",
  "tlogReplicas":"0"}
Live Nodes: [127.0.0.1:34901_solr, 127.0.0.1:34937_solr, 127.0.0.1:36053_solr, 127.0.0.1:43167_solr]
Last available state: DocCollection(collection1/39)={
  "pullReplicas":"0",
  "replicationFactor":"4",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node8":{
          "core":"collection1_shard1_replica_n7",
          "leader":"true",
          "core_node_name":"core_node8",
          "base_url":"https://127.0.0.1:34937/solr",
          "node_name":"127.0.0.1:34937_solr",
          "state":"active",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node10":{
          "core":"collection1_shard1_replica_n9",
          "core_node_name":"core_node10",
          "base_url":"https://127.0.0.1:36053/solr",
          "node_name":"127.0.0.1:36053_solr",
          "state":"recovering",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"4",
  "tlogReplicas":"0"}

Stack Trace:
java.lang.RuntimeException: Failed while waiting for active collection
Timeout waiting to see state for collection=collection1 :DocCollection(collection1/39)={
  "pullReplicas":"0",
  "replicationFactor":"4",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node8":{
          "core":"collection1_shard1_replica_n7",
          "leader":"true",
          "core_node_name":"core_node8",
          "base_url":"https://127.0.0.1:34937/solr",
          "node_name":"127.0.0.1:34937_solr",
          "state":"active",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node10":{
          "core":"collection1_shard1_replica_n9",
          "core_node_name":"core_node10",
          "base_url":"https://127.0.0.1:36053/solr",
          "node_name":"127.0.0.1:36053_solr",
          "state":"recovering",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"4",
  "tlogReplicas":"0"}
Live Nodes: [127.0.0.1:34901_solr, 127.0.0.1:34937_solr, 127.0.0.1:36053_solr, 127.0.0.1:43167_solr]
Last available state: DocCollection(collection1/39)={
  "pullReplicas":"0",
  "replicationFactor":"4",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node8":{
          "core":"collection1_shard1_replica_n7",
          "leader":"true",
          "core_node_name":"core_node8",
          "base_url":"https://127.0.0.1:34937/solr",
          "node_name":"127.0.0.1:34937_solr",
          "state":"active",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node10":{
          "core":"collection1_shard1_replica_n9",
          "core_node_name":"core_node10",
          "base_url":"https://127.0.0.1:36053/solr",
          "node_name":"127.0.0.1:36053_solr",
          "state":"recovering",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"4",
  "tlogReplicas":"0"}
	at __randomizedtesting.SeedInfo.seed([766B2CE6E90FD7C6:FE3F133C47F3BA3E]:0)
	at org.apache.solr.cloud.MiniSolrCloudCluster.waitForActiveCollection(MiniSolrCloudCluster.java:774)
	at org.apache.solr.cloud.MiniSolrCloudCluster.waitForActiveCollection(MiniSolrCloudCluster.java:780)
	at org.apache.solr.cloud.AssignBackwardCompatibilityTest.test(AssignBackwardCompatibilityTest.java:95)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:567)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:835)


FAILED:  org.apache.solr.cloud.BasicDistributedZk2Test.test

Error Message:
java.lang.AssertionError: There are still nodes recovering - waited for 330SECONDS

Stack Trace:
java.lang.AssertionError: There are still nodes recovering - waited for 330SECONDS
	at __randomizedtesting.SeedInfo.seed([766B2CE6E90FD7C6:FE3F133C47F3BA3E]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:197)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:144)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:139)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:1051)
	at org.apache.solr.cloud.BasicDistributedZk2Test.brindDownShardIndexSomeDocsAndRecover(BasicDistributedZk2Test.java:362)
	at org.apache.solr.cloud.BasicDistributedZk2Test.test(BasicDistributedZk2Test.java:103)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:567)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1090)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1061)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:835)


FAILED:  org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest.test

Error Message:
java.util.concurrent.TimeoutException: Cluster state still in flux after 30 retry intervals.

Stack Trace:
java.util.concurrent.TimeoutException: Cluster state still in flux after 30 retry intervals.
	at __randomizedtesting.SeedInfo.seed([766B2CE6E90FD7C6:FE3F133C47F3BA3E]:0)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForThingsToLevelOut(AbstractFullDistribZkTestBase.java:1643)
	at org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest.test(ChaosMonkeySafeLeaderWithPullReplicasTest.java:198)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:567)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1090)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1061)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:835)


FAILED:  org.apache.solr.cloud.DeleteReplicaTest.raceConditionOnDeleteAndRegisterReplica

Error Message:
java.lang.RuntimeException: Failed while waiting for active collection
Timeout waiting to see state for collection=raceDeleteReplicaCollection :DocCollection(raceDeleteReplicaCollection/17)={
  "pullReplicas":"0",
  "replicationFactor":"2",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node4":{
          "core":"raceDeleteReplicaCollection_shard1_replica_n2",
          "core_node_name":"core_node4",
          "base_url":"http://127.0.0.1:38875/solr",
          "node_name":"127.0.0.1:38875_solr",
          "state":"recovering",
          "collection":"raceDeleteReplicaCollection",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node6":{
          "core":"raceDeleteReplicaCollection_shard1_replica_n5",
          "leader":"true",
          "core_node_name":"core_node6",
          "base_url":"http://127.0.0.1:44991/solr",
          "node_name":"127.0.0.1:44991_solr",
          "state":"active",
          "collection":"raceDeleteReplicaCollection",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"2",
  "tlogReplicas":"0"}
Live Nodes: [127.0.0.1:34011_solr, 127.0.0.1:38875_solr, 127.0.0.1:44991_solr, 127.0.0.1:46223_solr]
Last available state: DocCollection(raceDeleteReplicaCollection/17)={
  "pullReplicas":"0",
  "replicationFactor":"2",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node4":{
          "core":"raceDeleteReplicaCollection_shard1_replica_n2",
          "core_node_name":"core_node4",
          "base_url":"http://127.0.0.1:38875/solr",
          "node_name":"127.0.0.1:38875_solr",
          "state":"recovering",
          "collection":"raceDeleteReplicaCollection",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node6":{
          "core":"raceDeleteReplicaCollection_shard1_replica_n5",
          "leader":"true",
          "core_node_name":"core_node6",
          "base_url":"http://127.0.0.1:44991/solr",
          "node_name":"127.0.0.1:44991_solr",
          "state":"active",
          "collection":"raceDeleteReplicaCollection",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"2",
  "tlogReplicas":"0"}

Stack Trace:
java.lang.RuntimeException: Failed while waiting for active collection
Timeout waiting to see state for collection=raceDeleteReplicaCollection :DocCollection(raceDeleteReplicaCollection/17)={
  "pullReplicas":"0",
  "replicationFactor":"2",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node4":{
          "core":"raceDeleteReplicaCollection_shard1_replica_n2",
          "core_node_name":"core_node4",
          "base_url":"http://127.0.0.1:38875/solr",
          "node_name":"127.0.0.1:38875_solr",
          "state":"recovering",
          "collection":"raceDeleteReplicaCollection",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node6":{
          "core":"raceDeleteReplicaCollection_shard1_replica_n5",
          "leader":"true",
          "core_node_name":"core_node6",
          "base_url":"http://127.0.0.1:44991/solr",
          "node_name":"127.0.0.1:44991_solr",
          "state":"active",
          "collection":"raceDeleteReplicaCollection",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"2",
  "tlogReplicas":"0"}
Live Nodes: [127.0.0.1:34011_solr, 127.0.0.1:38875_solr, 127.0.0.1:44991_solr, 127.0.0.1:46223_solr]
Last available state: DocCollection(raceDeleteReplicaCollection/17)={
  "pullReplicas":"0",
  "replicationFactor":"2",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node4":{
          "core":"raceDeleteReplicaCollection_shard1_replica_n2",
          "core_node_name":"core_node4",
          "base_url":"http://127.0.0.1:38875/solr",
          "node_name":"127.0.0.1:38875_solr",
          "state":"recovering",
          "collection":"raceDeleteReplicaCollection",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node6":{
          "core":"raceDeleteReplicaCollection_shard1_replica_n5",
          "leader":"true",
          "core_node_name":"core_node6",
          "base_url":"http://127.0.0.1:44991/solr",
          "node_name":"127.0.0.1:44991_solr",
          "state":"active",
          "collection":"raceDeleteReplicaCollection",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"2",
  "tlogReplicas":"0"}
	at __randomizedtesting.SeedInfo.seed([766B2CE6E90FD7C6:1C7D4D3681FD9D0C]:0)
	at org.apache.solr.cloud.MiniSolrCloudCluster.waitForActiveCollection(MiniSolrCloudCluster.java:774)
	at org.apache.solr.cloud.MiniSolrCloudCluster.waitForActiveCollection(MiniSolrCloudCluster.java:780)
	at org.apache.solr.cloud.DeleteReplicaTest.raceConditionOnDeleteAndRegisterReplica(DeleteReplicaTest.java:385)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:567)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:835)


FAILED:  org.apache.solr.cloud.HttpPartitionTest.test

Error Message:
java.lang.AssertionError: Didn't see all replicas for shard shard1 in collDoRecoveryOnRestart come up within 130000 ms! ClusterState: {
  "control_collection":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{"core_node2":{
            "core":"control_collection_shard1_replica_n1",
            "leader":"true",
            "core_node_name":"core_node2",
            "base_url":"http://127.0.0.1:44759",
            "node_name":"127.0.0.1:44759_",
            "state":"active",
            "collection":"control_collection",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"},
  "collDoRecoveryOnRestart":{
    "pullReplicas":"0",
    "replicationFactor":"2",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"collDoRecoveryOnRestart_shard1_replica_n1",
            "leader":"true",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:44759",
            "node_name":"127.0.0.1:44759_",
            "state":"active",
            "collection":"collDoRecoveryOnRestart",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node4":{
            "core":"collDoRecoveryOnRestart_shard1_replica_n2",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:36121",
            "node_name":"127.0.0.1:36121_",
            "state":"recovering",
            "collection":"collDoRecoveryOnRestart",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"2",
    "tlogReplicas":"0"},
  "collection1":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{
      "shard1":{
        "range":"80000000-ffffffff",
        "state":"active",
        "replicas":{"core_node4":{
            "core":"collection1_shard1_replica_n2",
            "leader":"true",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:36121",
            "node_name":"127.0.0.1:36121_",
            "state":"active",
            "collection":"collection1",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}},
      "shard2":{
        "range":"0-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"collection1_shard2_replica_n1",
            "leader":"true",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:35267",
            "node_name":"127.0.0.1:35267_",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node6":{
            "core":"collection1_shard2_replica_n5",
            "core_node_name":"core_node6",
            "base_url":"http://127.0.0.1:33131",
            "node_name":"127.0.0.1:33131_",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"}}

Stack Trace:
java.lang.AssertionError: Didn't see all replicas for shard shard1 in collDoRecoveryOnRestart come up within 130000 ms! ClusterState: {
  "control_collection":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{"core_node2":{
            "core":"control_collection_shard1_replica_n1",
            "leader":"true",
            "core_node_name":"core_node2",
            "base_url":"http://127.0.0.1:44759",
            "node_name":"127.0.0.1:44759_",
            "state":"active",
            "collection":"control_collection",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"},
  "collDoRecoveryOnRestart":{
    "pullReplicas":"0",
    "replicationFactor":"2",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"collDoRecoveryOnRestart_shard1_replica_n1",
            "leader":"true",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:44759",
            "node_name":"127.0.0.1:44759_",
            "state":"active",
            "collection":"collDoRecoveryOnRestart",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node4":{
            "core":"collDoRecoveryOnRestart_shard1_replica_n2",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:36121",
            "node_name":"127.0.0.1:36121_",
            "state":"recovering",
            "collection":"collDoRecoveryOnRestart",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"2",
    "tlogReplicas":"0"},
  "collection1":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{
      "shard1":{
        "range":"80000000-ffffffff",
        "state":"active",
        "replicas":{"core_node4":{
            "core":"collection1_shard1_replica_n2",
            "leader":"true",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:36121",
            "node_name":"127.0.0.1:36121_",
            "state":"active",
            "collection":"collection1",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}},
      "shard2":{
        "range":"0-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"collection1_shard2_replica_n1",
            "leader":"true",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:35267",
            "node_name":"127.0.0.1:35267_",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node6":{
            "core":"collection1_shard2_replica_n5",
            "core_node_name":"core_node6",
            "base_url":"http://127.0.0.1:33131",
            "node_name":"127.0.0.1:33131_",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"}}
	at __randomizedtesting.SeedInfo.seed([766B2CE6E90FD7C6:FE3F133C47F3BA3E]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.ensureAllReplicasAreActive(AbstractFullDistribZkTestBase.java:2147)
	at org.apache.solr.cloud.HttpPartitionTest.testDoRecoveryOnRestart(HttpPartitionTest.java:200)
	at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:132)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:567)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1090)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1061)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:835)


FAILED:  org.apache.solr.cloud.HttpPartitionWithTlogReplicasTest.test

Error Message:
java.lang.AssertionError: Didn't see all replicas for shard shard1 in collDoRecoveryOnRestart come up within 130000 ms! ClusterState: {
  "control_collection":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{"core_node2":{
            "core":"control_collection_shard1_replica_n1",
            "leader":"true",
            "core_node_name":"core_node2",
            "base_url":"http://127.0.0.1:43707",
            "node_name":"127.0.0.1:43707_",
            "state":"active",
            "collection":"control_collection",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"},
  "collDoRecoveryOnRestart":{
    "pullReplicas":"0",
    "replicationFactor":"0",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"collDoRecoveryOnRestart_shard1_replica_t1",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:39423",
            "node_name":"127.0.0.1:39423_",
            "state":"recovering",
            "collection":"collDoRecoveryOnRestart",
            "shard":"shard1",
            "type":"TLOG",
            "force_set_state":"false"},
          "core_node4":{
            "core":"collDoRecoveryOnRestart_shard1_replica_t2",
            "leader":"true",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:43707",
            "node_name":"127.0.0.1:43707_",
            "state":"active",
            "collection":"collDoRecoveryOnRestart",
            "shard":"shard1",
            "type":"TLOG",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"0",
    "tlogReplicas":"2"},
  "collection1":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{
      "shard1":{
        "range":"80000000-ffffffff",
        "state":"active",
        "replicas":{"core_node4":{
            "core":"collection1_shard1_replica_t2",
            "leader":"true",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:41211",
            "node_name":"127.0.0.1:41211_",
            "state":"active",
            "collection":"collection1",
            "shard":"shard1",
            "type":"TLOG",
            "force_set_state":"false"}}},
      "shard2":{
        "range":"0-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"collection1_shard2_replica_t1",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:39423",
            "node_name":"127.0.0.1:39423_",
            "state":"recovering",
            "collection":"collection1",
            "shard":"shard2",
            "type":"TLOG",
            "force_set_state":"false"},
          "core_node6":{
            "core":"collection1_shard2_replica_t5",
            "leader":"true",
            "core_node_name":"core_node6",
            "base_url":"http://127.0.0.1:43601",
            "node_name":"127.0.0.1:43601_",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"TLOG",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"}}

Stack Trace:
java.lang.AssertionError: Didn't see all replicas for shard shard1 in collDoRecoveryOnRestart come up within 130000 ms! ClusterState: {
  "control_collection":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{"core_node2":{
            "core":"control_collection_shard1_replica_n1",
            "leader":"true",
            "core_node_name":"core_node2",
            "base_url":"http://127.0.0.1:43707",
            "node_name":"127.0.0.1:43707_",
            "state":"active",
            "collection":"control_collection",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"},
  "collDoRecoveryOnRestart":{
    "pullReplicas":"0",
    "replicationFactor":"0",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"collDoRecoveryOnRestart_shard1_replica_t1",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:39423",
            "node_name":"127.0.0.1:39423_",
            "state":"recovering",
            "collection":"collDoRecoveryOnRestart",
            "shard":"shard1",
            "type":"TLOG",
            "force_set_state":"false"},
          "core_node4":{
            "core":"collDoRecoveryOnRestart_shard1_replica_t2",
            "leader":"true",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:43707",
            "node_name":"127.0.0.1:43707_",
            "state":"active",
            "collection":"collDoRecoveryOnRestart",
            "shard":"shard1",
            "type":"TLOG",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"0",
    "tlogReplicas":"2"},
  "collection1":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{
      "shard1":{
        "range":"80000000-ffffffff",
        "state":"active",
        "replicas":{"core_node4":{
            "core":"collection1_shard1_replica_t2",
            "leader":"true",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:41211",
            "node_name":"127.0.0.1:41211_",
            "state":"active",
            "collection":"collection1",
            "shard":"shard1",
            "type":"TLOG",
            "force_set_state":"false"}}},
      "shard2":{
        "range":"0-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"collection1_shard2_replica_t1",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:39423",
            "node_name":"127.0.0.1:39423_",
            "state":"recovering",
            "collection":"collection1",
            "shard":"shard2",
            "type":"TLOG",
            "force_set_state":"false"},
          "core_node6":{
            "core":"collection1_shard2_replica_t5",
            "leader":"true",
            "core_node_name":"core_node6",
            "base_url":"http://127.0.0.1:43601",
            "node_name":"127.0.0.1:43601_",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"TLOG",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"}}
	at __randomizedtesting.SeedInfo.seed([766B2CE6E90FD7C6:FE3F133C47F3BA3E]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.ensureAllReplicasAreActive(AbstractFullDistribZkTestBase.java:2147)
	at org.apache.solr.cloud.HttpPartitionTest.testDoRecoveryOnRestart(HttpPartitionTest.java:200)
	at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:132)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:567)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1090)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1061)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:835)


FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test

Error Message:
java.lang.AssertionError: Didn't see all replicas for shard shard1 in c8n_1x3_lf come up within 90000 ms! ClusterState: {
  "control_collection":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{"core_node2":{
            "core":"control_collection_shard1_replica_n1",
            "leader":"true",
            "core_node_name":"core_node2",
            "base_url":"http://127.0.0.1:44619",
            "node_name":"127.0.0.1:44619_",
            "state":"active",
            "collection":"control_collection",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"},
  "c8n_1x3_lf":{
    "pullReplicas":"0",
    "replicationFactor":"3",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"c8n_1x3_lf_shard1_replica_n1",
            "leader":"true",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:46667",
            "node_name":"127.0.0.1:46667_",
            "state":"active",
            "collection":"c8n_1x3_lf",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node5":{
            "core":"c8n_1x3_lf_shard1_replica_n2",
            "core_node_name":"core_node5",
            "base_url":"http://127.0.0.1:44619",
            "node_name":"127.0.0.1:44619_",
            "state":"recovering",
            "collection":"c8n_1x3_lf",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node6":{
            "core":"c8n_1x3_lf_shard1_replica_n4",
            "core_node_name":"core_node6",
            "base_url":"http://127.0.0.1:44627",
            "node_name":"127.0.0.1:44627_",
            "state":"recovering",
            "collection":"c8n_1x3_lf",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"3",
    "tlogReplicas":"0"},
  "collection1":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{
      "shard1":{
        "range":"80000000-ffffffff",
        "state":"active",
        "replicas":{"core_node4":{
            "core":"collection1_shard1_replica_n2",
            "leader":"true",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:38553",
            "node_name":"127.0.0.1:38553_",
            "state":"active",
            "collection":"collection1",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}},
      "shard2":{
        "range":"0-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"collection1_shard2_replica_n1",
            "leader":"true",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:46667",
            "node_name":"127.0.0.1:46667_",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node6":{
            "core":"collection1_shard2_replica_n5",
            "core_node_name":"core_node6",
            "base_url":"http://127.0.0.1:44627",
            "node_name":"127.0.0.1:44627_",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"}}

Stack Trace:
java.lang.AssertionError: Didn't see all replicas for shard shard1 in c8n_1x3_lf come up within 90000 ms! ClusterState: {
  "control_collection":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{"core_node2":{
            "core":"control_collection_shard1_replica_n1",
            "leader":"true",
            "core_node_name":"core_node2",
            "base_url":"http://127.0.0.1:44619",
            "node_name":"127.0.0.1:44619_",
            "state":"active",
            "collection":"control_collection",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"},
  "c8n_1x3_lf":{
    "pullReplicas":"0",
    "replicationFactor":"3",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"c8n_1x3_lf_shard1_replica_n1",
            "leader":"true",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:46667",
            "node_name":"127.0.0.1:46667_",
            "state":"active",
            "collection":"c8n_1x3_lf",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node5":{
            "core":"c8n_1x3_lf_shard1_replica_n2",
            "core_node_name":"core_node5",
            "base_url":"http://127.0.0.1:44619",
            "node_name":"127.0.0.1:44619_",
            "state":"recovering",
            "collection":"c8n_1x3_lf",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node6":{
            "core":"c8n_1x3_lf_shard1_replica_n4",
            "core_node_name":"core_node6",
            "base_url":"http://127.0.0.1:44627",
            "node_name":"127.0.0.1:44627_",
            "state":"recovering",
            "collection":"c8n_1x3_lf",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"3",
    "tlogReplicas":"0"},
  "collection1":{
    "pullReplicas":"0",
    "replicationFactor":"1",
    "shards":{
      "shard1":{
        "range":"80000000-ffffffff",
        "state":"active",
        "replicas":{"core_node4":{
            "core":"collection1_shard1_replica_n2",
            "leader":"true",
            "core_node_name":"core_node4",
            "base_url":"http://127.0.0.1:38553",
            "node_name":"127.0.0.1:38553_",
            "state":"active",
            "collection":"collection1",
            "shard":"shard1",
            "type":"NRT",
            "force_set_state":"false"}}},
      "shard2":{
        "range":"0-7fffffff",
        "state":"active",
        "replicas":{
          "core_node3":{
            "core":"collection1_shard2_replica_n1",
            "leader":"true",
            "core_node_name":"core_node3",
            "base_url":"http://127.0.0.1:46667",
            "node_name":"127.0.0.1:46667_",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"NRT",
            "force_set_state":"false"},
          "core_node6":{
            "core":"collection1_shard2_replica_n5",
            "core_node_name":"core_node6",
            "base_url":"http://127.0.0.1:44627",
            "node_name":"127.0.0.1:44627_",
            "state":"active",
            "collection":"collection1",
            "shard":"shard2",
            "type":"NRT",
            "force_set_state":"false"}}}},
    "router":{"name":"compositeId"},
    "nrtReplicas":"1",
    "tlogReplicas":"0"}}
	at __randomizedtesting.SeedInfo.seed([766B2CE6E90FD7C6:FE3F133C47F3BA3E]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.ensureAllReplicasAreActive(AbstractFullDistribZkTestBase.java:2147)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:101)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:58)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:567)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1090)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1061)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:835)


FAILED:  org.apache.solr.cloud.LeaderFailureAfterFreshStartTest.test

Error Message:
java.lang.AssertionError: timeout waiting to see all nodes active

Stack Trace:
java.lang.AssertionError: timeout waiting to see all nodes active
	at __randomizedtesting.SeedInfo.seed([766B2CE6E90FD7C6:FE3F133C47F3BA3E]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.solr.cloud.LeaderFailureAfterFreshStartTest.waitTillNodesActive(LeaderFailureAfterFreshStartTest.java:234)
	at org.apache.solr.cloud.LeaderFailureAfterFreshStartTest.restartNodes(LeaderFailureAfterFreshStartTest.java:174)
	at org.apache.solr.cloud.LeaderFailureAfterFreshStartTest.test(LeaderFailureAfterFreshStartTest.java:149)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:567)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1090)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1061)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:835)


FAILED:  org.apache.solr.cloud.LeaderVoteWaitTimeoutTest.testMostInSyncReplicasCanWinElection

Error Message:
java.lang.AssertionError: Timeout waiting for 1x3 collection
Timeout waiting to see state for collection=collection1 :DocCollection(collection1/26)={
  "pullReplicas":"0",
  "replicationFactor":"3",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node2":{
          "core":"collection1_shard1_replica_n1",
          "core_node_name":"core_node2",
          "base_url":"https://127.0.0.1:42741/solr",
          "node_name":"127.0.0.1:42741_solr",
          "state":"recovering",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node4":{
          "core":"collection1_shard1_replica_n3",
          "core_node_name":"core_node4",
          "base_url":"https://127.0.0.1:43247/solr",
          "node_name":"127.0.0.1:43247_solr",
          "state":"active",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node6":{
          "core":"collection1_shard1_replica_n5",
          "leader":"true",
          "core_node_name":"core_node6",
          "base_url":"https://127.0.0.1:46319/solr",
          "node_name":"127.0.0.1:46319_solr",
          "state":"active",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"3",
  "tlogReplicas":"0"}
Live Nodes: [127.0.0.1:34679_solr, 127.0.0.1:42741_solr, 127.0.0.1:43247_solr, 127.0.0.1:46319_solr]
Last available state: DocCollection(collection1/26)={
  "pullReplicas":"0",
  "replicationFactor":"3",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node2":{
          "core":"collection1_shard1_replica_n1",
          "core_node_name":"core_node2",
          "base_url":"https://127.0.0.1:42741/solr",
          "node_name":"127.0.0.1:42741_solr",
          "state":"recovering",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node4":{
          "core":"collection1_shard1_replica_n3",
          "core_node_name":"core_node4",
          "base_url":"https://127.0.0.1:43247/solr",
          "node_name":"127.0.0.1:43247_solr",
          "state":"active",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node6":{
          "core":"collection1_shard1_replica_n5",
          "leader":"true",
          "core_node_name":"core_node6",
          "base_url":"https://127.0.0.1:46319/solr",
          "node_name":"127.0.0.1:46319_solr",
          "state":"active",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"3",
  "tlogReplicas":"0"}

Stack Trace:
java.lang.AssertionError: Timeout waiting for 1x3 collection
Timeout waiting to see state for collection=collection1 :DocCollection(collection1/26)={
  "pullReplicas":"0",
  "replicationFactor":"3",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node2":{
          "core":"collection1_shard1_replica_n1",
          "core_node_name":"core_node2",
          "base_url":"https://127.0.0.1:42741/solr",
          "node_name":"127.0.0.1:42741_solr",
          "state":"recovering",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node4":{
          "core":"collection1_shard1_replica_n3",
          "core_node_name":"core_node4",
          "base_url":"https://127.0.0.1:43247/solr",
          "node_name":"127.0.0.1:43247_solr",
          "state":"active",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node6":{
          "core":"collection1_shard1_replica_n5",
          "leader":"true",
          "core_node_name":"core_node6",
          "base_url":"https://127.0.0.1:46319/solr",
          "node_name":"127.0.0.1:46319_solr",
          "state":"active",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"3",
  "tlogReplicas":"0"}
Live Nodes: [127.0.0.1:34679_solr, 127.0.0.1:42741_solr, 127.0.0.1:43247_solr, 127.0.0.1:46319_solr]
Last available state: DocCollection(collection1/26)={
  "pullReplicas":"0",
  "replicationFactor":"3",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node2":{
          "core":"collection1_shard1_replica_n1",
          "core_node_name":"core_node2",
          "base_url":"https://127.0.0.1:42741/solr",
          "node_name":"127.0.0.1:42741_solr",
          "state":"recovering",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node4":{
          "core":"collection1_shard1_replica_n3",
          "core_node_name":"core_node4",
          "base_url":"https://127.0.0.1:43247/solr",
          "node_name":"127.0.0.1:43247_solr",
          "state":"active",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node6":{
          "core":"collection1_shard1_replica_n5",
          "leader":"true",
          "core_node_name":"core_node6",
          "base_url":"https://127.0.0.1:46319/solr",
          "node_name":"127.0.0.1:46319_solr",
          "state":"active",
          "collection":"collection1",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"}}}},
  "router":{"name":"compositeId"},
  "nrtReplicas":"3",
  "tlogReplicas":"0"}
	at __randomizedtesting.SeedInfo.seed([766B2CE6E90FD7C6:DE77305C2B4FE3EC]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.apache.solr.cloud.SolrCloudTestCase.waitForState(SolrCloudTestCase.java:326)
	at org.apache.solr.cloud.SolrCloudTestCase.waitForState(SolrCloudTestCase.java:303)
	at org.apache.solr.cloud.LeaderVoteWaitTimeoutTest.testMostInSyncReplicasCanWinElection(LeaderVoteWaitTimeoutTest.java:253)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:567)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:835)


FAILED:  org.apache.solr.cloud.MigrateRouteKeyTest.multipleShardMigrateTest

Error Message:
java.lang.AssertionError: DocCount on target collection does not match expected:<19> but was:<16>

Stack Trace:
java.lang.AssertionError: DocCount on target collection does not match expected:<19> but was:<16>
	at __randomizedtesting.SeedInfo.seed([766B2CE6E90FD7C6:6A7B369CC8CF6727]:0)
	at org.junit.Assert.fail(Assert.java:89)
	at org.junit.Assert.failNotEquals(Assert.java:835)
	at org.junit.Assert.assertEquals(Assert.java:647)
	at org.apache.solr.cloud.MigrateRouteKeyTest.multipleShardMigrateTest(MigrateRouteKeyTest.java:159)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:567)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.junit.rules.RunRules.evaluate(RunRules.java:20)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxF

[...truncated too long message...]

nfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseG1GC" -Ptests.seed=766B2CE6E90FD7C6 -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=US-ASCII

  - org.apache.solr.cloud.MissingSegmentRecoveryTest.testLeaderRecovery (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.MissingSegmentRecoveryTest.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.MissingSegmentRecoveryTest" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseG1GC" -Ptests.seed=766B2CE6E90FD7C6 -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=US-ASCII

  - org.apache.solr.cloud.LeaderVoteWaitTimeoutTest.testMostInSyncReplicasCanWinElection (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.LeaderVoteWaitTimeoutTest.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.LeaderVoteWaitTimeoutTest" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseG1GC" -Ptests.seed=766B2CE6E90FD7C6 -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=US-ASCII

  - org.apache.solr.cloud.SyncSliceTest.test (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.SyncSliceTest.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.SyncSliceTest" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseG1GC" -Ptests.seed=766B2CE6E90FD7C6 -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=US-ASCII

  - org.apache.solr.cloud.HttpPartitionTest.test (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.HttpPartitionTest.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.HttpPartitionTest" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseG1GC" -Ptests.seed=766B2CE6E90FD7C6 -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=US-ASCII

  - org.apache.solr.cloud.UnloadDistributedZkTest.test (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.UnloadDistributedZkTest.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.UnloadDistributedZkTest" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseG1GC" -Ptests.seed=766B2CE6E90FD7C6 -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=US-ASCII

  - org.apache.solr.cloud.TestCloudRecovery.corruptedLogTest (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.TestCloudRecovery.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.TestCloudRecovery" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseG1GC" -Ptests.seed=766B2CE6E90FD7C6 -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=US-ASCII

  - org.apache.solr.cloud.MigrateRouteKeyTest.multipleShardMigrateTest (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.MigrateRouteKeyTest.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.MigrateRouteKeyTest" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseG1GC" -Ptests.seed=766B2CE6E90FD7C6 -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=US-ASCII

  - org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.RecoveryAfterSoftCommitTest.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.RecoveryAfterSoftCommitTest" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseG1GC" -Ptests.seed=766B2CE6E90FD7C6 -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=US-ASCII

  - org.apache.solr.cloud.DeleteReplicaTest.raceConditionOnDeleteAndRegisterReplica (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.DeleteReplicaTest.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.DeleteReplicaTest" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseG1GC" -Ptests.seed=766B2CE6E90FD7C6 -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=US-ASCII

  - org.apache.solr.cloud.TestLeaderElectionWithEmptyReplica.test (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.TestLeaderElectionWithEmptyReplica.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.TestLeaderElectionWithEmptyReplica" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseG1GC" -Ptests.seed=766B2CE6E90FD7C6 -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=US-ASCII

  - org.apache.solr.core.snapshots.TestSolrCloudSnapshots.testSnapshots (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.core.snapshots.TestSolrCloudSnapshots.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.core.snapshots.TestSolrCloudSnapshots" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseG1GC" -Ptests.seed=766B2CE6E90FD7C6 -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=US-ASCII

  - org.apache.solr.cloud.MoveReplicaTest.test (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.MoveReplicaTest.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.MoveReplicaTest" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseG1GC" -Ptests.seed=766B2CE6E90FD7C6 -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=US-ASCII

  - org.apache.solr.cloud.RecoveryZkTest.test (:solr:core)
    Test output: /home/jenkins/workspace/Lucene-Solr-master-Linux/solr/core/build/test-results/test/outputs/OUTPUT-org.apache.solr.cloud.RecoveryZkTest.txt
    Reproduce with: gradlew :solr:core:test --tests "org.apache.solr.cloud.RecoveryZkTest" -Ptests.jvms=6 -Ptests.haltonfailure=false "-Ptests.jvmargs=-XX:-UseCompressedOops -XX:+UseG1GC" -Ptests.seed=766B2CE6E90FD7C6 -Ptests.multiplier=3 -Ptests.badapples=false -Ptests.file.encoding=US-ASCII


Deprecated Gradle features were used in this build, making it incompatible with Gradle 7.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/6.6.1/userguide/command_line_interface.html#sec:command_line_warnings

BUILD SUCCESSFUL in 1h 4m 45s
854 actionable tasks: 854 executed
Build step 'Invoke Gradle script' changed build result to SUCCESS
Archiving artifacts
java.lang.InterruptedException: no matches found within 10000
	at hudson.FilePath$ValidateAntFileMask.hasMatch(FilePath.java:2875)
	at hudson.FilePath$ValidateAntFileMask.invoke(FilePath.java:2754)
	at hudson.FilePath$ValidateAntFileMask.invoke(FilePath.java:2735)
	at hudson.FilePath.act(FilePath.java:1076)
	at hudson.FilePath.act(FilePath.java:1059)
	at hudson.FilePath.validateAntFileMask(FilePath.java:2733)
	at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:270)
	at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:80)
	at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
	at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:804)
	at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:753)
	at hudson.model.Build$BuildExecution.post2(Build.java:177)
	at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:698)
	at hudson.model.Run.execute(Run.java:1919)
	at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
	at hudson.model.ResourceController.execute(ResourceController.java:97)
	at hudson.model.Executor.run(Executor.java:428)
No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/*_pid*.log". Configuration error?
[Java] Sleeping for 5 seconds due to JENKINS-32191...
[Java] Parsing console log (workspace: '/home/jenkins/workspace/Lucene-Solr-master-Linux')
[Java] Successfully parsed console log
[Java] -> found 0 issues (skipped 0 duplicates)
[Java] Skipping post processing
[Java] No filter has been set, publishing all 0 issues
[Java] Using reference build 'Lucene-Solr-master-Linux #28358' to compute new, fixed, and outstanding issues
[Java] Issues delta (vs. reference build): outstanding: 0, new: 0, fixed: 0
[Java] No quality gates have been set - skipping
[Java] Health report is disabled - skipping
[Java] Created analysis result for 0 issues (found 0 new issues, fixed 0 issues)
[Java] Attaching ResultAction with ID 'java' to run 'Lucene-Solr-master-Linux #28359'.
[Checks API] No suitable checks publisher found.
Recording test results
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
[Checks API] No suitable checks publisher found.
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)