You are viewing a plain text version of this content. The canonical link for it is here.
Posted to builds@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2020/08/08 06:25:35 UTC

[JENKINS] Lucene » Lucene-Solr-NightlyTests-master - Build # 2 - Unstable!

Build: https://ci-builds.apache.org/job/Lucene/job/Lucene-Solr-NightlyTests-master/2/

4 tests failed.
FAILED:  org.apache.solr.cloud.api.collections.TestHdfsCloudBackupRestore.test

Error Message:
Node 127.0.0.1:40407_solr has 6 replicas. Expected num replicas : 3. state:  DocCollection(hdfsbackuprestore_testok_restored/19)={   "pullReplicas":0,   "replicationFactor":2,   "shards":{     "shard2":{       "range":"0-7fffffff",       "state":"active",       "replicas":{         "core_node2":{           "core":"hdfsbackuprestore_testok_restored_shard2_replica_n1",           "leader":"true",           "core_node_name":"core_node2",           "base_url":"https://127.0.0.1:40407/solr",           "node_name":"127.0.0.1:40407_solr",           "state":"active",           "collection":"hdfsbackuprestore_testok_restored",           "shard":"shard2",           "type":"NRT",           "force_set_state":"false"},         "core_node8":{           "core":"hdfsbackuprestore_testok_restored_shard2_replica_n7",           "core_node_name":"core_node8",           "base_url":"https://127.0.0.1:40407/solr",           "node_name":"127.0.0.1:40407_solr",           "state":"active",           "collection":"hdfsbackuprestore_testok_restored",           "shard":"shard2",           "type":"NRT",           "force_set_state":"false"}},       "stateTimestamp":"1596862826832653725"},     "shard1_1":{       "range":"c0000000-ffffffff",       "state":"active",       "replicas":{         "core_node4":{           "core":"hdfsbackuprestore_testok_restored_shard1_1_replica_n3",           "leader":"true",           "core_node_name":"core_node4",           "base_url":"https://127.0.0.1:40407/solr",           "node_name":"127.0.0.1:40407_solr",           "state":"active",           "collection":"hdfsbackuprestore_testok_restored",           "shard":"shard1_1",           "type":"NRT",           "force_set_state":"false"},         "core_node10":{           "core":"hdfsbackuprestore_testok_restored_shard1_1_replica_n9",           "core_node_name":"core_node10",           "base_url":"https://127.0.0.1:40407/solr",           "node_name":"127.0.0.1:40407_solr",           "state":"active",           "collection":"hdfsbackuprestore_testok_restored",           "shard":"shard1_1",           "type":"NRT",           "force_set_state":"false"}},       "stateTimestamp":"1596862826832676634"},     "shard1_0":{       "range":"80000000-bfffffff",       "state":"active",       "replicas":{         "core_node6":{           "core":"hdfsbackuprestore_testok_restored_shard1_0_replica_n5",           "leader":"true",           "core_node_name":"core_node6",           "base_url":"https://127.0.0.1:40407/solr",           "node_name":"127.0.0.1:40407_solr",           "state":"active",           "collection":"hdfsbackuprestore_testok_restored",           "shard":"shard1_0",           "type":"NRT",           "force_set_state":"false"},         "core_node12":{           "core":"hdfsbackuprestore_testok_restored_shard1_0_replica_n11",           "core_node_name":"core_node12",           "base_url":"https://127.0.0.1:40407/solr",           "node_name":"127.0.0.1:40407_solr",           "state":"active",           "collection":"hdfsbackuprestore_testok_restored",           "shard":"shard1_0",           "type":"NRT",           "force_set_state":"false"}},       "stateTimestamp":"1596862826832692159"}},   "router":{"name":"compositeId"},   "nrtReplicas":2,   "tlogReplicas":0}

Stack Trace:
java.lang.AssertionError: Node 127.0.0.1:40407_solr has 6 replicas. Expected num replicas : 3. state: 
DocCollection(hdfsbackuprestore_testok_restored/19)={
  "pullReplicas":0,
  "replicationFactor":2,
  "shards":{
    "shard2":{
      "range":"0-7fffffff",
      "state":"active",
      "replicas":{
        "core_node2":{
          "core":"hdfsbackuprestore_testok_restored_shard2_replica_n1",
          "leader":"true",
          "core_node_name":"core_node2",
          "base_url":"https://127.0.0.1:40407/solr",
          "node_name":"127.0.0.1:40407_solr",
          "state":"active",
          "collection":"hdfsbackuprestore_testok_restored",
          "shard":"shard2",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node8":{
          "core":"hdfsbackuprestore_testok_restored_shard2_replica_n7",
          "core_node_name":"core_node8",
          "base_url":"https://127.0.0.1:40407/solr",
          "node_name":"127.0.0.1:40407_solr",
          "state":"active",
          "collection":"hdfsbackuprestore_testok_restored",
          "shard":"shard2",
          "type":"NRT",
          "force_set_state":"false"}},
      "stateTimestamp":"1596862826832653725"},
    "shard1_1":{
      "range":"c0000000-ffffffff",
      "state":"active",
      "replicas":{
        "core_node4":{
          "core":"hdfsbackuprestore_testok_restored_shard1_1_replica_n3",
          "leader":"true",
          "core_node_name":"core_node4",
          "base_url":"https://127.0.0.1:40407/solr",
          "node_name":"127.0.0.1:40407_solr",
          "state":"active",
          "collection":"hdfsbackuprestore_testok_restored",
          "shard":"shard1_1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node10":{
          "core":"hdfsbackuprestore_testok_restored_shard1_1_replica_n9",
          "core_node_name":"core_node10",
          "base_url":"https://127.0.0.1:40407/solr",
          "node_name":"127.0.0.1:40407_solr",
          "state":"active",
          "collection":"hdfsbackuprestore_testok_restored",
          "shard":"shard1_1",
          "type":"NRT",
          "force_set_state":"false"}},
      "stateTimestamp":"1596862826832676634"},
    "shard1_0":{
      "range":"80000000-bfffffff",
      "state":"active",
      "replicas":{
        "core_node6":{
          "core":"hdfsbackuprestore_testok_restored_shard1_0_replica_n5",
          "leader":"true",
          "core_node_name":"core_node6",
          "base_url":"https://127.0.0.1:40407/solr",
          "node_name":"127.0.0.1:40407_solr",
          "state":"active",
          "collection":"hdfsbackuprestore_testok_restored",
          "shard":"shard1_0",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node12":{
          "core":"hdfsbackuprestore_testok_restored_shard1_0_replica_n11",
          "core_node_name":"core_node12",
          "base_url":"https://127.0.0.1:40407/solr",
          "node_name":"127.0.0.1:40407_solr",
          "state":"active",
          "collection":"hdfsbackuprestore_testok_restored",
          "shard":"shard1_0",
          "type":"NRT",
          "force_set_state":"false"}},
      "stateTimestamp":"1596862826832692159"}},
  "router":{"name":"compositeId"},
  "nrtReplicas":2,
  "tlogReplicas":0}
	at __randomizedtesting.SeedInfo.seed([707BAB5045BCF7AA:F82F948AEB409A52]:0)
	at org.junit.Assert.fail(Assert.java:88)
	at org.junit.Assert.assertTrue(Assert.java:41)
	at org.apache.solr.cloud.api.collections.AbstractCloudBackupRestoreTestCase.lambda$testBackupAndRestore$1(AbstractCloudBackupRestoreTestCase.java:374)
	at java.base/java.util.HashMap.forEach(HashMap.java:1336)
	at org.apache.solr.cloud.api.collections.AbstractCloudBackupRestoreTestCase.testBackupAndRestore(AbstractCloudBackupRestoreTestCase.java:373)
	at org.apache.solr.cloud.api.collections.AbstractCloudBackupRestoreTestCase.test(AbstractCloudBackupRestoreTestCase.java:155)
	at org.apache.solr.cloud.api.collections.TestHdfsCloudBackupRestore.test(TestHdfsCloudBackupRestore.java:225)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:834)


FAILED:  org.apache.solr.cloud.api.collections.TestLocalFSCloudBackupRestore.test

Error Message:
Node 127.0.0.1:41663_solr has 6 replicas. Expected num replicas : 3. state:  DocCollection(backuprestore_testok_restored/15)={   "pullReplicas":0,   "replicationFactor":2,   "shards":{     "shard2":{       "range":"0-7fffffff",       "state":"active",       "replicas":{         "core_node2":{           "core":"backuprestore_testok_restored_shard2_replica_n1",           "leader":"true",           "core_node_name":"core_node2",           "base_url":"https://127.0.0.1:41663/solr",           "node_name":"127.0.0.1:41663_solr",           "state":"active",           "collection":"backuprestore_testok_restored",           "shard":"shard2",           "type":"NRT",           "force_set_state":"false"},         "core_node8":{           "core":"backuprestore_testok_restored_shard2_replica_n7",           "core_node_name":"core_node8",           "base_url":"https://127.0.0.1:41663/solr",           "node_name":"127.0.0.1:41663_solr",           "state":"active",           "collection":"backuprestore_testok_restored",           "shard":"shard2",           "type":"NRT",           "force_set_state":"false"}},       "stateTimestamp":"1596865190834118125"},     "shard1_1":{       "range":"c0000000-ffffffff",       "state":"active",       "replicas":{         "core_node4":{           "core":"backuprestore_testok_restored_shard1_1_replica_n3",           "leader":"true",           "core_node_name":"core_node4",           "base_url":"https://127.0.0.1:41663/solr",           "node_name":"127.0.0.1:41663_solr",           "state":"active",           "collection":"backuprestore_testok_restored",           "shard":"shard1_1",           "type":"NRT",           "force_set_state":"false"},         "core_node10":{           "core":"backuprestore_testok_restored_shard1_1_replica_n9",           "core_node_name":"core_node10",           "base_url":"https://127.0.0.1:41663/solr",           "node_name":"127.0.0.1:41663_solr",           "state":"active",           "collection":"backuprestore_testok_restored",           "shard":"shard1_1",           "type":"NRT",           "force_set_state":"false"}},       "stateTimestamp":"1596865190834141029"},     "shard1_0":{       "range":"80000000-bfffffff",       "state":"active",       "replicas":{         "core_node6":{           "core":"backuprestore_testok_restored_shard1_0_replica_n5",           "leader":"true",           "core_node_name":"core_node6",           "base_url":"https://127.0.0.1:41663/solr",           "node_name":"127.0.0.1:41663_solr",           "state":"active",           "collection":"backuprestore_testok_restored",           "shard":"shard1_0",           "type":"NRT",           "force_set_state":"false"},         "core_node12":{           "core":"backuprestore_testok_restored_shard1_0_replica_n11",           "core_node_name":"core_node12",           "base_url":"https://127.0.0.1:41663/solr",           "node_name":"127.0.0.1:41663_solr",           "state":"active",           "collection":"backuprestore_testok_restored",           "shard":"shard1_0",           "type":"NRT",           "force_set_state":"false"}},       "stateTimestamp":"1596865190834156887"}},   "router":{"name":"compositeId"},   "nrtReplicas":2,   "tlogReplicas":0}

Stack Trace:
java.lang.AssertionError: Node 127.0.0.1:41663_solr has 6 replicas. Expected num replicas : 3. state: 
DocCollection(backuprestore_testok_restored/15)={
  "pullReplicas":0,
  "replicationFactor":2,
  "shards":{
    "shard2":{
      "range":"0-7fffffff",
      "state":"active",
      "replicas":{
        "core_node2":{
          "core":"backuprestore_testok_restored_shard2_replica_n1",
          "leader":"true",
          "core_node_name":"core_node2",
          "base_url":"https://127.0.0.1:41663/solr",
          "node_name":"127.0.0.1:41663_solr",
          "state":"active",
          "collection":"backuprestore_testok_restored",
          "shard":"shard2",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node8":{
          "core":"backuprestore_testok_restored_shard2_replica_n7",
          "core_node_name":"core_node8",
          "base_url":"https://127.0.0.1:41663/solr",
          "node_name":"127.0.0.1:41663_solr",
          "state":"active",
          "collection":"backuprestore_testok_restored",
          "shard":"shard2",
          "type":"NRT",
          "force_set_state":"false"}},
      "stateTimestamp":"1596865190834118125"},
    "shard1_1":{
      "range":"c0000000-ffffffff",
      "state":"active",
      "replicas":{
        "core_node4":{
          "core":"backuprestore_testok_restored_shard1_1_replica_n3",
          "leader":"true",
          "core_node_name":"core_node4",
          "base_url":"https://127.0.0.1:41663/solr",
          "node_name":"127.0.0.1:41663_solr",
          "state":"active",
          "collection":"backuprestore_testok_restored",
          "shard":"shard1_1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node10":{
          "core":"backuprestore_testok_restored_shard1_1_replica_n9",
          "core_node_name":"core_node10",
          "base_url":"https://127.0.0.1:41663/solr",
          "node_name":"127.0.0.1:41663_solr",
          "state":"active",
          "collection":"backuprestore_testok_restored",
          "shard":"shard1_1",
          "type":"NRT",
          "force_set_state":"false"}},
      "stateTimestamp":"1596865190834141029"},
    "shard1_0":{
      "range":"80000000-bfffffff",
      "state":"active",
      "replicas":{
        "core_node6":{
          "core":"backuprestore_testok_restored_shard1_0_replica_n5",
          "leader":"true",
          "core_node_name":"core_node6",
          "base_url":"https://127.0.0.1:41663/solr",
          "node_name":"127.0.0.1:41663_solr",
          "state":"active",
          "collection":"backuprestore_testok_restored",
          "shard":"shard1_0",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node12":{
          "core":"backuprestore_testok_restored_shard1_0_replica_n11",
          "core_node_name":"core_node12",
          "base_url":"https://127.0.0.1:41663/solr",
          "node_name":"127.0.0.1:41663_solr",
          "state":"active",
          "collection":"backuprestore_testok_restored",
          "shard":"shard1_0",
          "type":"NRT",
          "force_set_state":"false"}},
      "stateTimestamp":"1596865190834156887"}},
  "router":{"name":"compositeId"},
  "nrtReplicas":2,
  "tlogReplicas":0}
	at __randomizedtesting.SeedInfo.seed([707BAB5045BCF7AA:F82F948AEB409A52]:0)
	at org.junit.Assert.fail(Assert.java:88)
	at org.junit.Assert.assertTrue(Assert.java:41)
	at org.apache.solr.cloud.api.collections.AbstractCloudBackupRestoreTestCase.lambda$testBackupAndRestore$1(AbstractCloudBackupRestoreTestCase.java:374)
	at java.base/java.util.HashMap.forEach(HashMap.java:1336)
	at org.apache.solr.cloud.api.collections.AbstractCloudBackupRestoreTestCase.testBackupAndRestore(AbstractCloudBackupRestoreTestCase.java:373)
	at org.apache.solr.cloud.api.collections.AbstractCloudBackupRestoreTestCase.test(AbstractCloudBackupRestoreTestCase.java:155)
	at org.apache.solr.cloud.api.collections.TestLocalFSCloudBackupRestore.test(TestLocalFSCloudBackupRestore.java:93)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:834)


FAILED:  org.apache.solr.servlet.TestRequestRateLimiter.testConcurrentQueries

Error Message:
Incoming accepted new request count did not match. Expected 5 incoming 25

Stack Trace:
java.lang.AssertionError: Incoming accepted new request count did not match. Expected 5 incoming 25
	at __randomizedtesting.SeedInfo.seed([707BAB5045BCF7AA:32E04875C5DB828B]:0)
	at org.junit.Assert.fail(Assert.java:88)
	at org.junit.Assert.assertTrue(Assert.java:41)
	at org.apache.solr.servlet.TestRequestRateLimiter.testConcurrentQueries(TestRequestRateLimiter.java:74)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:834)


FAILED:  org.apache.solr.servlet.TestRequestRateLimiter.testSlotBorrowing

Error Message:
Incoming slots borrowed count did not match. Expected > 0  incoming 0

Stack Trace:
java.lang.AssertionError: Incoming slots borrowed count did not match. Expected > 0  incoming 0
	at __randomizedtesting.SeedInfo.seed([707BAB5045BCF7AA:C16C4D6F2AEA0C0E]:0)
	at org.junit.Assert.fail(Assert.java:88)
	at org.junit.Assert.assertTrue(Assert.java:41)
	at org.apache.solr.servlet.TestRequestRateLimiter.testSlotBorrowing(TestRequestRateLimiter.java:106)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:834)




Build Log:
[...truncated 12718 lines...]
   [junit4] Suite: org.apache.solr.cloud.api.collections.TestHdfsCloudBackupRestore
   [junit4]   2> 1300338 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[707BAB5045BCF7AA]-worker) [     ] o.a.s.SolrTestCase Setting 'solr.default.confdir' system property to test-framework derived value of '/home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/server/solr/configsets/_default/conf'
   [junit4]   2> 1300339 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[707BAB5045BCF7AA]-worker) [     ] o.a.s.SolrTestCaseJ4 Created dataDir: /home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.TestHdfsCloudBackupRestore_707BAB5045BCF7AA-001/data-dir-58-001
   [junit4]   2> 1300339 WARN  (SUITE-TestHdfsCloudBackupRestore-seed#[707BAB5045BCF7AA]-worker) [     ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=10 numCloses=10
   [junit4]   2> 1300339 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[707BAB5045BCF7AA]-worker) [     ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 1300340 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[707BAB5045BCF7AA]-worker) [     ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason="", value=0.0/0.0, ssl=0.0/0.0, clientAuth=0.0/0.0)
   [junit4]   2> 1300340 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[707BAB5045BCF7AA]-worker) [     ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   1> Formatting using clusterid: testClusterID
   [junit4]   2> 1300423 WARN  (SUITE-TestHdfsCloudBackupRestore-seed#[707BAB5045BCF7AA]-worker) [     ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 1300425 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[707BAB5045BCF7AA]-worker) [     ] o.e.j.s.Server jetty-9.4.27.v20200227; built: 2020-02-27T18:37:21.340Z; git: a304fd9f351f337e7c0e2a7c28878dd536149c6c; jvm 11.0.6+8-LTS
   [junit4]   2> 1300428 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[707BAB5045BCF7AA]-worker) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 1300428 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[707BAB5045BCF7AA]-worker) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 1300428 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[707BAB5045BCF7AA]-worker) [     ] o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 1300428 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[707BAB5045BCF7AA]-worker) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@263246fe{static,/static,jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.0-tests.jar!/webapps/static,AVAILABLE}
   [junit4]   2> 1300582 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[707BAB5045BCF7AA]-worker) [     ] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@50572719{hdfs,/,file:///home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/jetty-localhost_localdomain-39113-hadoop-hdfs-3_2_0-tests_jar-_-any-15182565898331055083.dir/webapp/,AVAILABLE}{jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.0-tests.jar!/webapps/hdfs}
   [junit4]   2> 1300582 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[707BAB5045BCF7AA]-worker) [     ] o.e.j.s.AbstractConnector Started ServerConnector@6ed59e15{HTTP/1.1, (http/1.1)}{localhost.localdomain:39113}
   [junit4]   2> 1300582 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[707BAB5045BCF7AA]-worker) [     ] o.e.j.s.Server Started @1300623ms
   [junit4]   2> 1300645 WARN  (SUITE-TestHdfsCloudBackupRestore-seed#[707BAB5045BCF7AA]-worker) [     ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 1300646 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[707BAB5045BCF7AA]-worker) [     ] o.e.j.s.Server jetty-9.4.27.v20200227; built: 2020-02-27T18:37:21.340Z; git: a304fd9f351f337e7c0e2a7c28878dd536149c6c; jvm 11.0.6+8-LTS
   [junit4]   2> 1300647 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[707BAB5045BCF7AA]-worker) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 1300647 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[707BAB5045BCF7AA]-worker) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 1300647 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[707BAB5045BCF7AA]-worker) [     ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 1300647 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[707BAB5045BCF7AA]-worker) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@351a36db{static,/static,jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.0-tests.jar!/webapps/static,AVAILABLE}
   [junit4]   2> 1300756 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[707BAB5045BCF7AA]-worker) [     ] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@b91c70f{datanode,/,file:///home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/jetty-localhost-34837-hadoop-hdfs-3_2_0-tests_jar-_-any-7520589488631302440.dir/webapp/,AVAILABLE}{jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.0-tests.jar!/webapps/datanode}
   [junit4]   2> 1300757 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[707BAB5045BCF7AA]-worker) [     ] o.e.j.s.AbstractConnector Started ServerConnector@5f51d08f{HTTP/1.1, (http/1.1)}{localhost:34837}
   [junit4]   2> 1300757 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[707BAB5045BCF7AA]-worker) [     ] o.e.j.s.Server Started @1300798ms
   [junit4]   2> 1300779 WARN  (SUITE-TestHdfsCloudBackupRestore-seed#[707BAB5045BCF7AA]-worker) [     ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 1300780 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[707BAB5045BCF7AA]-worker) [     ] o.e.j.s.Server jetty-9.4.27.v20200227; built: 2020-02-27T18:37:21.340Z; git: a304fd9f351f337e7c0e2a7c28878dd536149c6c; jvm 11.0.6+8-LTS
   [junit4]   2> 1300781 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[707BAB5045BCF7AA]-worker) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 1300781 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[707BAB5045BCF7AA]-worker) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 1300781 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[707BAB5045BCF7AA]-worker) [     ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 1300781 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[707BAB5045BCF7AA]-worker) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6dc6dd6{static,/static,jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.0-tests.jar!/webapps/static,AVAILABLE}
   [junit4]   2> 1300888 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[707BAB5045BCF7AA]-worker) [     ] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@161b6c8c{datanode,/,file:///home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/jetty-localhost-37403-hadoop-hdfs-3_2_0-tests_jar-_-any-6364898589314091680.dir/webapp/,AVAILABLE}{jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.0-tests.jar!/webapps/datanode}
   [junit4]   2> 1300888 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[707BAB5045BCF7AA]-worker) [     ] o.e.j.s.AbstractConnector Started ServerConnector@7f37273b{HTTP/1.1, (http/1.1)}{localhost:37403}
   [junit4]   2> 1300888 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[707BAB5045BCF7AA]-worker) [     ] o.e.j.s.Server Started @1300929ms
   [junit4]   2> 1300917 INFO  (Block report processor) [     ] BlockStateChange BLOCK* processReport 0xc7f885b57ed0c9bc: Processing first storage report for DS-f8676bf6-6c0a-4b5a-8b38-26a00e3c767c from datanode f7b0a94e-50ac-46d4-8200-b57c6e86b0f5
   [junit4]   2> 1300917 INFO  (Block report processor) [     ] BlockStateChange BLOCK* processReport 0xc7f885b57ed0c9bc: from storage DS-f8676bf6-6c0a-4b5a-8b38-26a00e3c767c node DatanodeRegistration(127.0.0.1:37155, datanodeUuid=f7b0a94e-50ac-46d4-8200-b57c6e86b0f5, infoPort=37921, infoSecurePort=0, ipcPort=45647, storageInfo=lv=-57;cid=testClusterID;nsid=1125638140;c=1596862805090), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
   [junit4]   2> 1300918 INFO  (Block report processor) [     ] BlockStateChange BLOCK* processReport 0xc7f885b57ed0c9bc: Processing first storage report for DS-cfbdf838-21c8-4766-8da0-9802b2ea91db from datanode f7b0a94e-50ac-46d4-8200-b57c6e86b0f5
   [junit4]   2> 1300918 INFO  (Block report processor) [     ] BlockStateChange BLOCK* processReport 0xc7f885b57ed0c9bc: from storage DS-cfbdf838-21c8-4766-8da0-9802b2ea91db node DatanodeRegistration(127.0.0.1:37155, datanodeUuid=f7b0a94e-50ac-46d4-8200-b57c6e86b0f5, infoPort=37921, infoSecurePort=0, ipcPort=45647, storageInfo=lv=-57;cid=testClusterID;nsid=1125638140;c=1596862805090), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
   [junit4]   2> 1301037 INFO  (Block report processor) [     ] BlockStateChange BLOCK* processReport 0x97010945d4ee7b2c: Processing first storage report for DS-f1ff925c-e71f-40c0-9c55-a2989af1c456 from datanode 8509957a-f657-4e7d-b2ea-d1444016d51c
   [junit4]   2> 1301038 INFO  (Block report processor) [     ] BlockStateChange BLOCK* processReport 0x97010945d4ee7b2c: from storage DS-f1ff925c-e71f-40c0-9c55-a2989af1c456 node DatanodeRegistration(127.0.0.1:46225, datanodeUuid=8509957a-f657-4e7d-b2ea-d1444016d51c, infoPort=36959, infoSecurePort=0, ipcPort=36599, storageInfo=lv=-57;cid=testClusterID;nsid=1125638140;c=1596862805090), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
   [junit4]   2> 1301038 INFO  (Block report processor) [     ] BlockStateChange BLOCK* processReport 0x97010945d4ee7b2c: Processing first storage report for DS-d3107148-0d7a-4b4b-804f-be28fbe3f657 from datanode 8509957a-f657-4e7d-b2ea-d1444016d51c
   [junit4]   2> 1301038 INFO  (Block report processor) [     ] BlockStateChange BLOCK* processReport 0x97010945d4ee7b2c: from storage DS-d3107148-0d7a-4b4b-804f-be28fbe3f657 node DatanodeRegistration(127.0.0.1:46225, datanodeUuid=8509957a-f657-4e7d-b2ea-d1444016d51c, infoPort=36959, infoSecurePort=0, ipcPort=36599, storageInfo=lv=-57;cid=testClusterID;nsid=1125638140;c=1596862805090), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
   [junit4]   2> 1301124 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[707BAB5045BCF7AA]-worker) [     ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.TestHdfsCloudBackupRestore_707BAB5045BCF7AA-001/tempDir-002
   [junit4]   2> 1301124 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[707BAB5045BCF7AA]-worker) [     ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 1301125 INFO  (ZkTestServer Run Thread) [     ] o.a.s.c.ZkTestServer client port: 0.0.0.0/0.0.0.0:0
   [junit4]   2> 1301125 INFO  (ZkTestServer Run Thread) [     ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 1301125 WARN  (ZkTestServer Run Thread) [     ] o.a.z.s.ServerCnxnFactory maxCnxns is not configured, using default value 0.
   [junit4]   2> 1301225 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[707BAB5045BCF7AA]-worker) [     ] o.a.s.c.ZkTestServer start zk server on port: 38149
   [junit4]   2> 1301225 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[707BAB5045BCF7AA]-worker) [     ] o.a.s.c.ZkTestServer waitForServerUp: 127.0.0.1:38149
   [junit4]   2> 1301225 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[707BAB5045BCF7AA]-worker) [     ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:38149
   [junit4]   2> 1301225 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[707BAB5045BCF7AA]-worker) [     ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 38149
   [junit4]   2> 1301226 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[707BAB5045BCF7AA]-worker) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1301228 INFO  (zkConnectionManagerCallback-10833-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1301228 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[707BAB5045BCF7AA]-worker) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1301231 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[707BAB5045BCF7AA]-worker) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1301232 INFO  (zkConnectionManagerCallback-10835-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1301232 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[707BAB5045BCF7AA]-worker) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1301232 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[707BAB5045BCF7AA]-worker) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1301233 INFO  (zkConnectionManagerCallback-10837-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1301233 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[707BAB5045BCF7AA]-worker) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1301337 INFO  (jetty-launcher-10838-thread-1) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
   [junit4]   2> 1301337 INFO  (jetty-launcher-10838-thread-2) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
   [junit4]   2> 1301338 INFO  (jetty-launcher-10838-thread-1) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 2 ...
   [junit4]   2> 1301338 INFO  (jetty-launcher-10838-thread-2) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 2 ...
   [junit4]   2> 1301338 INFO  (jetty-launcher-10838-thread-1) [     ] o.e.j.s.Server jetty-9.4.27.v20200227; built: 2020-02-27T18:37:21.340Z; git: a304fd9f351f337e7c0e2a7c28878dd536149c6c; jvm 11.0.6+8-LTS
   [junit4]   2> 1301338 INFO  (jetty-launcher-10838-thread-2) [     ] o.e.j.s.Server jetty-9.4.27.v20200227; built: 2020-02-27T18:37:21.340Z; git: a304fd9f351f337e7c0e2a7c28878dd536149c6c; jvm 11.0.6+8-LTS
   [junit4]   2> 1301339 INFO  (jetty-launcher-10838-thread-2) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 1301339 INFO  (jetty-launcher-10838-thread-2) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 1301339 INFO  (jetty-launcher-10838-thread-1) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 1301339 INFO  (jetty-launcher-10838-thread-2) [     ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 1301339 INFO  (jetty-launcher-10838-thread-1) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 1301339 INFO  (jetty-launcher-10838-thread-1) [     ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 1301340 INFO  (jetty-launcher-10838-thread-2) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7dd0b756{/solr,null,AVAILABLE}
   [junit4]   2> 1301340 INFO  (jetty-launcher-10838-thread-1) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1646f5d1{/solr,null,AVAILABLE}
   [junit4]   2> 1301340 INFO  (jetty-launcher-10838-thread-2) [     ] o.e.j.s.AbstractConnector Started ServerConnector@48befe54{ssl, (ssl, alpn, http/1.1, h2)}{127.0.0.1:40407}
   [junit4]   2> 1301341 INFO  (jetty-launcher-10838-thread-2) [     ] o.e.j.s.Server Started @1301381ms
   [junit4]   2> 1301341 INFO  (jetty-launcher-10838-thread-2) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=40407}
   [junit4]   2> 1301341 INFO  (jetty-launcher-10838-thread-1) [     ] o.e.j.s.AbstractConnector Started ServerConnector@27b93181{ssl, (ssl, alpn, http/1.1, h2)}{127.0.0.1:45567}
   [junit4]   2> 1301341 INFO  (jetty-launcher-10838-thread-1) [     ] o.e.j.s.Server Started @1301382ms
   [junit4]   2> 1301341 INFO  (jetty-launcher-10838-thread-1) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=45567}
   [junit4]   2> 1301341 ERROR (jetty-launcher-10838-thread-2) [     ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1301341 INFO  (jetty-launcher-10838-thread-2) [     ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 1301341 INFO  (jetty-launcher-10838-thread-2) [     ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 9.0.0
   [junit4]   2> 1301341 INFO  (jetty-launcher-10838-thread-2) [     ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1301341 ERROR (jetty-launcher-10838-thread-1) [     ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1301341 INFO  (jetty-launcher-10838-thread-2) [     ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: /home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr
   [junit4]   2> 1301341 INFO  (jetty-launcher-10838-thread-1) [     ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 1301341 INFO  (jetty-launcher-10838-thread-2) [     ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2020-08-08T05:00:06.069419Z
   [junit4]   2> 1301341 INFO  (jetty-launcher-10838-thread-1) [     ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 9.0.0
   [junit4]   2> 1301341 INFO  (jetty-launcher-10838-thread-1) [     ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1301341 INFO  (jetty-launcher-10838-thread-1) [     ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: /home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr
   [junit4]   2> 1301341 INFO  (jetty-launcher-10838-thread-1) [     ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2020-08-08T05:00:06.069541Z
   [junit4]   2> 1301342 INFO  (jetty-launcher-10838-thread-2) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1301343 INFO  (jetty-launcher-10838-thread-1) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1301343 INFO  (zkConnectionManagerCallback-10840-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1301343 INFO  (jetty-launcher-10838-thread-2) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1301343 INFO  (zkConnectionManagerCallback-10842-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1301343 INFO  (jetty-launcher-10838-thread-1) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1301343 INFO  (jetty-launcher-10838-thread-2) [     ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1301343 INFO  (jetty-launcher-10838-thread-1) [     ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1301347 INFO  (jetty-launcher-10838-thread-2) [     ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@3a179179, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 1301352 INFO  (jetty-launcher-10838-thread-1) [     ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@3a179179, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 1301830 INFO  (jetty-launcher-10838-thread-2) [     ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
   [junit4]   2> 1301831 WARN  (jetty-launcher-10838-thread-2) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@23dde5ac[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1301831 WARN  (jetty-launcher-10838-thread-2) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@23dde5ac[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1301834 WARN  (jetty-launcher-10838-thread-2) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@5b8ddac1[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1301834 WARN  (jetty-launcher-10838-thread-2) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@5b8ddac1[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1301836 INFO  (jetty-launcher-10838-thread-2) [     ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:38149/solr
   [junit4]   2> 1301837 INFO  (jetty-launcher-10838-thread-2) [n:127.0.0.1:40407_solr     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1301839 INFO  (zkConnectionManagerCallback-10855-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1301839 INFO  (jetty-launcher-10838-thread-2) [n:127.0.0.1:40407_solr     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1301916 INFO  (jetty-launcher-10838-thread-2) [n:127.0.0.1:40407_solr     ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:40407_solr
   [junit4]   2> 1301917 INFO  (jetty-launcher-10838-thread-2) [n:127.0.0.1:40407_solr     ] o.a.s.c.Overseer Overseer (id=72274539187994629-127.0.0.1:40407_solr-n_0000000000) starting
   [junit4]   2> 1301924 INFO  (OverseerStateUpdate-72274539187994629-127.0.0.1:40407_solr-n_0000000000) [n:127.0.0.1:40407_solr     ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:40407_solr
   [junit4]   2> 1301924 INFO  (jetty-launcher-10838-thread-2) [n:127.0.0.1:40407_solr     ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:40407_solr
   [junit4]   2> 1301925 INFO  (OverseerStateUpdate-72274539187994629-127.0.0.1:40407_solr-n_0000000000) [n:127.0.0.1:40407_solr     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1301926 INFO  (jetty-launcher-10838-thread-2) [n:127.0.0.1:40407_solr     ] o.a.s.p.PackageLoader /packages.json updated to version -1
   [junit4]   2> 1301927 WARN  (jetty-launcher-10838-thread-2) [n:127.0.0.1:40407_solr     ] o.a.s.c.CoreContainer Not all security plugins configured!  authentication=disabled authorization=disabled.  Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external.  See https://s.apache.org/solrsecurity for more info
   [junit4]   2> 1301927 INFO  (jetty-launcher-10838-thread-2) [n:127.0.0.1:40407_solr     ] o.a.s.c.b.r.BackupRepositoryFactory Added backup repository with configuration params {type = repository,name = hdfs,class = org.apache.solr.core.backup.repository.HdfsBackupRepository,attributes = {name=hdfs, class=org.apache.solr.core.backup.repository.HdfsBackupRepository},args = {location=/backup, solr.hdfs.home=hdfs://localhost.localdomain:46613/solr, solr.hdfs.confdir=}}
   [junit4]   2> 1301927 INFO  (jetty-launcher-10838-thread-2) [n:127.0.0.1:40407_solr     ] o.a.s.c.b.r.BackupRepositoryFactory Added backup repository with configuration params {type = repository,name = poisioned,class = org.apache.solr.cloud.api.collections.TestLocalFSCloudBackupRestore$PoinsionedRepository,attributes = {default=true, name=poisioned, class=org.apache.solr.cloud.api.collections.TestLocalFSCloudBackupRestore$PoinsionedRepository},}
   [junit4]   2> 1301927 INFO  (jetty-launcher-10838-thread-2) [n:127.0.0.1:40407_solr     ] o.a.s.c.b.r.BackupRepositoryFactory Default configuration for backup repository is with configuration params {type = repository,name = poisioned,class = org.apache.solr.cloud.api.collections.TestLocalFSCloudBackupRestore$PoinsionedRepository,attributes = {default=true, name=poisioned, class=org.apache.solr.cloud.api.collections.TestLocalFSCloudBackupRestore$PoinsionedRepository},}
   [junit4]   2> 1301940 INFO  (jetty-launcher-10838-thread-2) [n:127.0.0.1:40407_solr     ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 1301957 INFO  (jetty-launcher-10838-thread-2) [n:127.0.0.1:40407_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3a179179
   [junit4]   2> 1301964 INFO  (jetty-launcher-10838-thread-2) [n:127.0.0.1:40407_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3a179179
   [junit4]   2> 1301964 INFO  (jetty-launcher-10838-thread-2) [n:127.0.0.1:40407_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3a179179
   [junit4]   2> 1301965 INFO  (jetty-launcher-10838-thread-2) [n:127.0.0.1:40407_solr     ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.TestHdfsCloudBackupRestore_707BAB5045BCF7AA-001/tempDir-002/node2
   [junit4]   2> 1302608 INFO  (jetty-launcher-10838-thread-1) [     ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
   [junit4]   2> 1302609 WARN  (jetty-launcher-10838-thread-1) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@4cf7fca9[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1302609 WARN  (jetty-launcher-10838-thread-1) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@4cf7fca9[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1302612 WARN  (jetty-launcher-10838-thread-1) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@49ed7460[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1302612 WARN  (jetty-launcher-10838-thread-1) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@49ed7460[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 1302613 INFO  (jetty-launcher-10838-thread-1) [     ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:38149/solr
   [junit4]   2> 1302614 INFO  (jetty-launcher-10838-thread-1) [n:127.0.0.1:45567_solr     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1302615 INFO  (zkConnectionManagerCallback-10872-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1302615 INFO  (jetty-launcher-10838-thread-1) [n:127.0.0.1:45567_solr     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1302618 INFO  (jetty-launcher-10838-thread-1) [n:127.0.0.1:45567_solr     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1302624 INFO  (jetty-launcher-10838-thread-1) [n:127.0.0.1:45567_solr     ] o.a.s.c.ZkController Publish node=127.0.0.1:45567_solr as DOWN
   [junit4]   2> 1302628 INFO  (jetty-launcher-10838-thread-1) [n:127.0.0.1:45567_solr     ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 1000 transient cores
   [junit4]   2> 1302628 INFO  (jetty-launcher-10838-thread-1) [n:127.0.0.1:45567_solr     ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:45567_solr
   [junit4]   2> 1302629 INFO  (zkCallback-10854-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1302629 INFO  (zkCallback-10871-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1302630 INFO  (jetty-launcher-10838-thread-1) [n:127.0.0.1:45567_solr     ] o.a.s.p.PackageLoader /packages.json updated to version -1
   [junit4]   2> 1302630 WARN  (jetty-launcher-10838-thread-1) [n:127.0.0.1:45567_solr     ] o.a.s.c.CoreContainer Not all security plugins configured!  authentication=disabled authorization=disabled.  Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external.  See https://s.apache.org/solrsecurity for more info
   [junit4]   2> 1302630 INFO  (jetty-launcher-10838-thread-1) [n:127.0.0.1:45567_solr     ] o.a.s.c.b.r.BackupRepositoryFactory Added backup repository with configuration params {type = repository,name = hdfs,class = org.apache.solr.core.backup.repository.HdfsBackupRepository,attributes = {name=hdfs, class=org.apache.solr.core.backup.repository.HdfsBackupRepository},args = {location=/backup, solr.hdfs.home=hdfs://localhost.localdomain:46613/solr, solr.hdfs.confdir=}}
   [junit4]   2> 1302630 INFO  (jetty-launcher-10838-thread-1) [n:127.0.0.1:45567_solr     ] o.a.s.c.b.r.BackupRepositoryFactory Added backup repository with configuration params {type = repository,name = poisioned,class = org.apache.solr.cloud.api.collections.TestLocalFSCloudBackupRestore$PoinsionedRepository,attributes = {default=true, name=poisioned, class=org.apache.solr.cloud.api.collections.TestLocalFSCloudBackupRestore$PoinsionedRepository},}
   [junit4]   2> 1302630 INFO  (jetty-launcher-10838-thread-1) [n:127.0.0.1:45567_solr     ] o.a.s.c.b.r.BackupRepositoryFactory Default configuration for backup repository is with configuration params {type = repository,name = poisioned,class = org.apache.solr.cloud.api.collections.TestLocalFSCloudBackupRestore$PoinsionedRepository,attributes = {default=true, name=poisioned, class=org.apache.solr.cloud.api.collections.TestLocalFSCloudBackupRestore$PoinsionedRepository},}
   [junit4]   2> 1302643 INFO  (jetty-launcher-10838-thread-1) [n:127.0.0.1:45567_solr     ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 1302660 INFO  (jetty-launcher-10838-thread-1) [n:127.0.0.1:45567_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3a179179
   [junit4]   2> 1302669 INFO  (jetty-launcher-10838-thread-1) [n:127.0.0.1:45567_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3a179179
   [junit4]   2> 1302669 INFO  (jetty-launcher-10838-thread-1) [n:127.0.0.1:45567_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3a179179
   [junit4]   2> 1302670 INFO  (jetty-launcher-10838-thread-1) [n:127.0.0.1:45567_solr     ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.TestHdfsCloudBackupRestore_707BAB5045BCF7AA-001/tempDir-002/node1
   [junit4]   2> 1302678 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[707BAB5045BCF7AA]-worker) [     ] o.a.s.c.MiniSolrCloudCluster waitForAllNodes: numServers=2
   [junit4]   2> 1302679 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[707BAB5045BCF7AA]-worker) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 1302679 INFO  (zkConnectionManagerCallback-10885-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1302679 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[707BAB5045BCF7AA]-worker) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 1302680 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[707BAB5045BCF7AA]-worker) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1302681 INFO  (SUITE-TestHdfsCloudBackupRestore-seed#[707BAB5045BCF7AA]-worker) [     ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:38149/solr ready
   [junit4]   2> 1302714 INFO  (TEST-TestHdfsCloudBackupRestore.testRestoreFailure-seed#[707BAB5045BCF7AA]) [     ] o.a.s.SolrTestCaseJ4 ###Starting testRestoreFailure
   [junit4]   2> 1302739 INFO  (qtp1111921956-18438) [n:127.0.0.1:40407_solr     ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params pullReplicas=1&collection.configName=conf1&name=hdfsbackuprestore_testfailure&nrtReplicas=1&action=CREATE&numShards=2&tlogReplicas=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1302741 INFO  (OverseerThreadFactory-10860-thread-1-processing-n:127.0.0.1:40407_solr) [n:127.0.0.1:40407_solr     ] o.a.s.c.a.c.CreateCollectionCmd Create collection hdfsbackuprestore_testfailure
   [junit4]   2> 1302844 WARN  (OverseerThreadFactory-10860-thread-1-processing-n:127.0.0.1:40407_solr) [n:127.0.0.1:40407_solr     ] o.a.s.c.a.c.CreateCollectionCmd Specified number of replicas of 3 on collection hdfsbackuprestore_testfailure is higher than the number of Solr instances currently live or live and part of your createNodeSet(2). It's unusual to run two replica of the same slice on the same Solr-instance.
   [junit4]   2> 1302848 INFO  (OverseerStateUpdate-72274539187994629-127.0.0.1:40407_solr-n_0000000000) [n:127.0.0.1:40407_solr     ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"hdfsbackuprestore_testfailure",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"hdfsbackuprestore_testfailure_shard1_replica_n1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:40407/solr",
   [junit4]   2>   "node_name":"127.0.0.1:40407_solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 1302850 INFO  (OverseerStateUpdate-72274539187994629-127.0.0.1:40407_solr-n_0000000000) [n:127.0.0.1:40407_solr     ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"hdfsbackuprestore_testfailure",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"hdfsbackuprestore_testfailure_shard1_replica_t2",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:45567/solr",
   [junit4]   2>   "node_name":"127.0.0.1:45567_solr",
   [junit4]   2>   "type":"TLOG",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 1302852 INFO  (OverseerStateUpdate-72274539187994629-127.0.0.1:40407_solr-n_0000000000) [n:127.0.0.1:40407_solr     ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"hdfsbackuprestore_testfailure",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"hdfsbackuprestore_testfailure_shard1_replica_p4",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:40407/solr",
   [junit4]   2>   "node_name":"127.0.0.1:40407_solr",
   [junit4]   2>   "type":"PULL",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 1302854 INFO  (OverseerStateUpdate-72274539187994629-127.0.0.1:40407_solr-n_0000000000) [n:127.0.0.1:40407_solr     ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"hdfsbackuprestore_testfailure",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"hdfsbackuprestore_testfailure_shard2_replica_n6",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:45567/solr",
   [junit4]   2>   "node_name":"127.0.0.1:45567_solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 1302856 INFO  (OverseerStateUpdate-72274539187994629-127.0.0.1:40407_solr-n_0000000000) [n:127.0.0.1:40407_solr     ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"hdfsbackuprestore_testfailure",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"hdfsbackuprestore_testfailure_shard2_replica_t8",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:40407/solr",
   [junit4]   2>   "node_name":"127.0.0.1:40407_solr",
   [junit4]   2>   "type":"TLOG",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 1302858 INFO  (OverseerStateUpdate-72274539187994629-127.0.0.1:40407_solr-n_0000000000) [n:127.0.0.1:40407_solr     ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"hdfsbackuprestore_testfailure",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"hdfsbackuprestore_testfailure_shard2_replica_p10",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:45567/solr",
   [junit4]   2>   "node_name":"127.0.0.1:45567_solr",
   [junit4]   2>   "type":"PULL",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 1303090 INFO  (qtp759257841-18436) [n:127.0.0.1:45567_solr    x:hdfsbackuprestore_testfailure_shard1_replica_t2 ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node5&collection.configName=conf1&newCollection=true&name=hdfsbackuprestore_testfailure_shard1_replica_t2&action=CREATE&numShards=2&collection=hdfsbackuprestore_testfailure&shard=shard1&wt=javabin&version=2&replicaType=TLOG
   [junit4]   2> 1303090 INFO  (qtp1111921956-18437) [n:127.0.0.1:40407_solr    x:hdfsbackuprestore_testfailure_shard1_replica_n1 ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf1&newCollection=true&name=hdfsbackuprestore_testfailure_shard1_replica_n1&action=CREATE&numShards=2&collection=hdfsbackuprestore_testfailure&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 1303090 INFO  (qtp759257841-18441) [n:127.0.0.1:45567_solr    x:hdfsbackuprestore_testfailure_shard2_replica_p10 ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node12&collection.configName=conf1&newCollection=true&name=hdfsbackuprestore_testfailure_shard2_replica_p10&action=CREATE&numShards=2&collection=hdfsbackuprestore_testfailure&shard=shard2&wt=javabin&version=2&replicaType=PULL
   [junit4]   2> 1303091 INFO  (qtp1111921956-18437) [n:127.0.0.1:40407_solr    x:hdfsbackuprestore_testfailure_shard1_replica_n1 ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 1000 transient cores
   [junit4]   2> 1303091 INFO  (qtp1111921956-18433) [n:127.0.0.1:40407_solr    x:hdfsbackuprestore_testfailure_shard2_replica_t8 ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node11&collection.configName=conf1&newCollection=true&name=hdfsbackuprestore_testfailure_shard2_replica_t8&action=CREATE&numShards=2&collection=hdfsbackuprestore_testfailure&shard=shard2&wt=javabin&version=2&replicaType=TLOG
   [junit4]   2> 1303091 INFO  (qtp759257841-18439) [n:127.0.0.1:45567_solr    x:hdfsbackuprestore_testfailure_shard2_replica_n6 ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node9&collection.configName=conf1&newCollection=true&name=hdfsbackuprestore_testfailure_shard2_replica_n6&action=CREATE&numShards=2&collection=hdfsbackuprestore_testfailure&shard=shard2&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 1303090 INFO  (qtp1111921956-18440) [n:127.0.0.1:40407_solr    x:hdfsbackuprestore_testfailure_shard1_replica_p4 ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node7&collection.configName=conf1&newCollection=true&name=hdfsbackuprestore_testfailure_shard1_replica_p4&action=CREATE&numShards=2&collection=hdfsbackuprestore_testfailure&shard=shard1&wt=javabin&version=2&replicaType=PULL
   [junit4]   2> 1303100 INFO  (qtp759257841-18436) [n:127.0.0.1:45567_solr c:hdfsbackuprestore_testfailure s:shard1 r:core_node5 x:hdfsbackuprestore_testfailure_shard1_replica_t2 ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.0.0
   [junit4]   2> 1303104 INFO  (qtp759257841-18439) [n:127.0.0.1:45567_solr c:hdfsbackuprestore_testfailure s:shard2 r:core_node9 x:hdfsbackuprestore_testfailure_shard2_replica_n6 ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.0.0
   [junit4]   2> 1303104 INFO  (qtp1111921956-18440) [n:127.0.0.1:40407_solr c:hdfsbackuprestore_testfailure s:shard1 r:core_node7 x:hdfsbackuprestore_testfailure_shard1_replica_p4 ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.0.0
   [junit4]   2> 1303106 INFO  (qtp759257841-18436) [n:127.0.0.1:45567_solr c:hdfsbackuprestore_testfailure s:shard1 r:core_node5 x:hdfsbackuprestore_testfailure_shard1_replica_t2 ] o.a.s.c.CoreContainer Creating SolrCore 'hdfsbackuprestore_testfailure_shard1_replica_t2' using configuration from configset conf1, trusted=true
   [junit4]   2> 1303120 INFO  (qtp1111921956-18433) [n:127.0.0.1:40407_solr c:hdfsbackuprestore_testfailure s:shard2 r:core_node11 x:hdfsbackuprestore_testfailure_shard2_replica_t8 ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.0.0
   [junit4]   2> 1303120 INFO  (qtp759257841-18439) [n:127.0.0.1:45567_solr c:hdfsbackuprestore_testfailure s:shard2 r:core_node9 x:hdfsbackuprestore_testfailure_shard2_replica_n6 ] o.a.s.c.CoreContainer Creating SolrCore 'hdfsbackuprestore_testfailure_shard2_replica_n6' using configuration from configset conf1, trusted=true
   [junit4]   2> 1303133 INFO  (qtp1111921956-18440) [n:127.0.0.1:40407_solr c:hdfsbackuprestore_testfailure s:shard1 r:core_node7 x:hdfsbackuprestore_testfailure_shard1_replica_p4 ] o.a.s.c.CoreContainer Creating SolrCore 'hdfsbackuprestore_testfailure_shard1_replica_p4' using configuration from configset conf1, trusted=true
   [junit4]   2> 1303135 INFO  (qtp759257841-18441) [n:127.0.0.1:45567_solr c:hdfsbackuprestore_testfailure s:shard2 r:core_node12 x:hdfsbackuprestore_testfailure_shard2_replica_p10 ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.0.0
   [junit4]   2> 1303135 INFO  (qtp759257841-18436) [n:127.0.0.1:45567_solr c:hdfsbackuprestore_testfailure s:shard1 r:core_node5 x:hdfsbackuprestore_testfailure_shard1_replica_t2 ] o.a.s.s.IndexSchema Schema name=minimal
   [junit4]   2> 1303136 INFO  (qtp1111921956-18440) [n:127.0.0.1:40407_solr c:hdfsbackuprestore_testfailure s:shard1 r:core_node7 x:hdfsbackuprestore_testfailure_shard1_replica_p4 ] o.a.s.s.IndexSchema Schema name=minimal
   [junit4]   2> 1303137 INFO  (qtp759257841-18436) [n:127.0.0.1:45567_solr c:hdfsbackuprestore_testfailure s:shard1 r:core_node5 x:hdfsbackuprestore_testfailure_shard1_replica_t2 ] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 1303137 INFO  (qtp759257841-18436) [n:127.0.0.1:45567_solr c:hdfsbackuprestore_testfailure s:shard1 r:core_node5 x:hdfsbackuprestore_testfailure_shard1_replica_t2 ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.hdfsbackuprestore_testfailure.shard1.replica_t2' (registry 'solr.core.hdfsbackuprestore_testfailure.shard1.replica_t2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3a179179
   [junit4]   2> 1303137 INFO  (qtp759257841-18436) [n:127.0.0.1:45567_solr c:hdfsbackuprestore_testfailure s:shard1 r:core_node5 x:hdfsbackuprestore_testfailure_shard1_replica_t2 ] o.a.s.c.SolrCore [[hdfsbackuprestore_testfailure_shard1_replica_t2] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.TestHdfsCloudBackupRestore_707BAB5045BCF7AA-001/tempDir-002/node1/hdfsbackuprestore_testfailure_shard1_replica_t2], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.TestHdfsCloudBackupRestore_707BAB5045BCF7AA-001/tempDir-002/node1/hdfsbackuprestore_testfailure_shard1_replica_t2/data/]
   [junit4]   2> 1303138 INFO  (qtp1111921956-18440) [n:127.0.0.1:40407_solr c:hdfsbackuprestore_testfailure s:shard1 r:core_node7 x:hdfsbackuprestore_testfailure_shard1_replica_p4 ] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 1303138 INFO  (qtp1111921956-18440) [n:127.0.0.1:40407_solr c:hdfsbackuprestore_testfailure s:shard1 r:core_node7 x:hdfsbackuprestore_testfailure_shard1_replica_p4 ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.hdfsbackuprestore_testfailure.shard1.replica_p4' (registry 'solr.core.hdfsbackuprestore_testfailure.shard1.replica_p4') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3a179179
   [junit4]   2> 1303138 INFO  (qtp1111921956-18440) [n:127.0.0.1:40407_solr c:hdfsbackuprestore_testfailure s:shard1 r:core_node7 x:hdfsbackuprestore_testfailure_shard1_replica_p4 ] o.a.s.c.SolrCore [[hdfsbackuprestore_testfailure_shard1_replica_p4] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.TestHdfsCloudBackupRestore_707BAB5045BCF7AA-001/tempDir-002/node2/hdfsbackuprestore_testfailure_shard1_replica_p4], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.TestHdfsCloudBackupRestore_707BAB5045BCF7AA-001/tempDir-002/node2/hdfsbackuprestore_testfailure_shard1_replica_p4/data/]
   [junit4]   2> 1303141 INFO  (qtp759257841-18439) [n:127.0.0.1:45567_solr c:hdfsbackuprestore_testfailure s:shard2 r:core_node9 x:hdfsbackuprestore_testfailure_shard2_replica_n6 ] o.a.s.s.IndexSchema Schema name=minimal
   [junit4]   2> 1303141 INFO  (qtp759257841-18441) [n:127.0.0.1:45567_solr c:hdfsbackuprestore_testfailure s:shard2 r:core_node12 x:hdfsbackuprestore_testfailure_shard2_replica_p10 ] o.a.s.c.CoreContainer Creating SolrCore 'hdfsbackuprestore_testfailure_shard2_replica_p10' using configuration from configset conf1, trusted=true
   [junit4]   2> 1303142 INFO  (qtp759257841-18439) [n:127.0.0.1:45567_solr c:hdfsbackuprestore_testfailure s:shard2 r:core_node9 x:hdfsbackuprestore_testfailure_shard2_replica_n6 ] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 1303142 INFO  (qtp759257841-18439) [n:127.0.0.1:45567_solr c:hdfsbackuprestore_testfailure s:shard2 r:core_node9 x:hdfsbackuprestore_testfailure_shard2_replica_n6 ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.hdfsbackuprestore_testfailure.shard2.replica_n6' (registry 'solr.core.hdfsbackuprestore_testfailure.shard2.replica_n6') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3a179179
   [junit4]   2> 1303142 INFO  (qtp759257841-18439) [n:127.0.0.1:45567_solr c:hdfsbackuprestore_testfailure s:shard2 r:core_node9 x:hdfsbackuprestore_testfailure_shard2_replica_n6 ] o.a.s.c.SolrCore [[hdfsbackuprestore_testfailure_shard2_replica_n6] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.TestHdfsCloudBackupRestore_707BAB5045BCF7AA-001/tempDir-002/node1/hdfsbackuprestore_testfailure_shard2_replica_n6], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.TestHdfsCloudBackupRestore_707BAB5045BCF7AA-001/tempDir-002/node1/hdfsbackuprestore_testfailure_shard2_replica_n6/data/]
   [junit4]   2> 1303146 INFO  (qtp759257841-18441) [n:127.0.0.1:45567_solr c:hdfsbackuprestore_testfailure s:shard2 r:core_node12 x:hdfsbackuprestore_testfailure_shard2_replica_p10 ] o.a.s.s.IndexSchema Schema name=minimal
   [junit4]   2> 1303149 INFO  (qtp1111921956-18437) [n:127.0.0.1:40407_solr c:hdfsbackuprestore_testfailure s:shard1 r:core_node3 x:hdfsbackuprestore_testfailure_shard1_replica_n1 ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.0.0
   [junit4]   2> 1303149 INFO  (qtp1111921956-18433) [n:127.0.0.1:40407_solr c:hdfsbackuprestore_testfailure s:shard2 r:core_node11 x:hdfsbackuprestore_testfailure_shard2_replica_t8 ] o.a.s.c.CoreContainer Creating SolrCore 'hdfsbackuprestore_testfailure_shard2_replica_t8' using configuration from configset conf1, trusted=true
   [junit4]   2> 1303152 INFO  (qtp759257841-18441) [n:127.0.0.1:45567_solr c:hdfsbackuprestore_testfailure s:shard2 r:core_node12 x:hdfsbackuprestore_testfailure_shard2_replica_p10 ] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 1303153 INFO  (qtp759257841-18441) [n:127.0.0.1:45567_solr c:hdfsbackuprestore_testfailure s:shard2 r:core_node12 x:hdfsbackuprestore_testfailure_shard2_replica_p10 ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.hdfsbackuprestore_testfailure.shard2.replica_p10' (registry 'solr.core.hdfsbackuprestore_testfailure.shard2.replica_p10') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3a179179
   [junit4]   2> 1303153 INFO  (qtp759257841-18441) [n:127.0.0.1:45567_solr c:hdfsbackuprestore_testfailure s:shard2 r:core_node12 x:hdfsbackuprestore_testfailure_shard2_replica_p10 ] o.a.s.c.SolrCore [[hdfsbackuprestore_testfailure_shard2_replica_p10] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.TestHdfsCloudBackupRestore_707BAB5045BCF7AA-001/tempDir-002/node1/hdfsbackuprestore_testfailure_shard2_replica_p10], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.TestHdfsCloudBackupRestore_707BAB5045BCF7AA-001/tempDir-002/node1/hdfsbackuprestore_testfailure_shard2_replica_p10/data/]
   [junit4]   2> 1303154 INFO  (qtp1111921956-18437) [n:127.0.0.1:40407_solr c:hdfsbackuprestore_testfailure s:shard1 r:core_node3 x:hdfsbackuprestore_testfailure_shard1_replica_n1 ] o.a.s.c.CoreContainer Creating SolrCore 'hdfsbackuprestore_testfailure_shard1_replica_n1' using configuration from configset conf1, trusted=true
   [junit4]   2> 1303156 INFO  (qtp1111921956-18433) [n:127.0.0.1:40407_solr c:hdfsbackuprestore_testfailure s:shard2 r:core_node11 x:hdfsbackuprestore_testfailure_shard2_replica_t8 ] o.a.s.s.IndexSchema Schema name=minimal
   [junit4]   2> 1303157 INFO  (qtp1111921956-18433) [n:127.0.0.1:40407_solr c:hdfsbackuprestore_testfailure s:shard2 r:core_node11 x:hdfsbackuprestore_testfailure_shard2_replica_t8 ] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 1303157 INFO  (qtp1111921956-18433) [n:127.0.0.1:40407_solr c:hdfsbackuprestore_testfailure s:shard2 r:core_node11 x:hdfsbackuprestore_testfailure_shard2_replica_t8 ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.hdfsbackuprestore_testfailure.shard2.replica_t8' (registry 'solr.core.hdfsbackuprestore_testfailure.shard2.replica_t8') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3a179179
   [junit4]   2> 1303157 INFO  (qtp1111921956-18433) [n:127.0.0.1:40407_solr c:hdfsbackuprestore_testfailure s:shard2 r:core_node11 x:hdfsbackuprestore_testfailure_shard2_replica_t8 ] o.a.s.c.SolrCore [[hdfsbackuprestore_testfailure_shard2_replica_t8] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.TestHdfsCloudBackupRestore_707BAB5045BCF7AA-001/tempDir-002/node2/hdfsbackuprestore_testfailure_shard2_replica_t8], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.TestHdfsCloudBackupRestore_707BAB5045BCF7AA-001/tempDir-002/node2/hdfsbackuprestore_testfailure_shard2_replica_t8/data/]
   [junit4]   2> 1303163 INFO  (qtp1111921956-18437) [n:127.0.0.1:40407_solr c:hdfsbackuprestore_testfailure s:shard1 r:core_node3 x:hdfsbackuprestore_testfailure_shard1_replica_n1 ] o.a.s.s.IndexSchema Schema name=minimal
   [junit4]   2> 1303164 INFO  (qtp1111921956-18437) [n:127.0.0.1:40407_solr c:hdfsbackuprestore_testfailure s:shard1 r:core_node3 x:hdfsbackuprestore_testfailure_shard1_replica_n1 ] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 1303164 INFO  (qtp1111921956-18437) [n:127.0.0.1:40407_solr c:hdfsbackuprestore_testfailure s:shard1 r:core_node3 x:hdfsbackuprestore_testfailure_shard1_replica_n1 ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.hdfsbackuprestore_testfailure.shard1.replica_n1' (registry 'solr.core.hdfsbackuprestore_testfailure.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3a179179
   [junit4]   2> 1303165 INFO  (qtp1111921956-18437) [n:127.0.0.1:40407_solr c:hdfsbackuprestore_testfailure s:shard1 r:core_node3 x:hdfsbackuprestore_testfailure_shard1_replica_n1 ] o.a.s.c.SolrCore [[hdfsbackuprestore_testfailure_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.TestHdfsCloudBackupRestore_707BAB5045BCF7AA-001/tempDir-002/node2/hdfsbackuprestore_testfailure_shard1_replica_n1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.api.collections.TestHdfsCloudBackupRestore_707BAB5045BCF7AA-001/tempDir-002/node2/hdfsbackuprestore_testfailure_shard1_replica_n1/data/]
   [junit4]   2> 1303255 INFO  (qtp759257841-18436) [n:127.0.0.1:45567_solr c:hdfsbackuprestore_testfailure s:shard1 r:core_node5 x:hdfsbackuprestore_testfailure_shard1_replica_t2 ] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1303255 INFO  (qtp759257841-18436) [n:127.0.0.1:45567_solr c:hdfsbackuprestore_testfailure s:shard1 r:core_node5 x:hdfsbackuprestore_testfailure_shard1_replica_t2 ] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1303255 INFO  (qtp759257841-18439) [n:127.0.0.1:45567_solr c:hdfsbackuprestore_testfailure s:shard2 r:core_node9 x:hdfsbackuprestore_testfailure_shard2_replica_n6 ] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1303255 INFO  (qtp759257841-18439) [n:127.0.0.1:45567_solr c:hdfsbackuprestore_testfailure s:shard2 r:core_node9 x:hdfsbackuprestore_testfailure_shard2_replica_n6 ] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1303269 INFO  (qtp1111921956-18433) [n:127.0.0.1:40407_solr c:hdfsbackuprestore_testfailure s:shard2 r:core_node11 x:hdfsbackuprestore_testfailure_shard2_replica_t8 ] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1303269 INFO  (qtp1111921956-18433) [n:127.0.0.1:40407_solr c:hdfsbackuprestore_testfailure s:shard2 r:core_node11 x:hdfsbackuprestore_testfailure_shard2_replica_t8 ] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1303270 INFO  (qtp759257841-18436) [n:127.0.0.1:45567_solr c:hdfsbackuprestore_testfailure s:shard1 r:core_node5 x:hdfsbackuprestore_testfailure_shard1_replica_t2 ] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1303270 INFO  (qtp759257841-18436) [n:127.0.0.1:45567_solr c:hdfsbackuprestore_testfailure s:shard1 r:core_node5 x:hdfsbackuprestore_testfailure_shard1_replica_t2 ] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1303271 INFO  (qtp1111921956-18433) [n:127.0.0.1:40407_solr c:hdfsbackuprestore_testfailure s:shard2 r:core_node11 x:hdfsbackuprestore_testfailure_shard2_replica_t8 ] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1303271 INFO  (qtp1111921956-18433) [n:127.0.0.1:40407_solr c:hdfsbackuprestore_testfailure s:shard2 r:core_node11 x:hdfsbackuprestore_testfailure_shard2_replica_t8 ] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1303272 INFO  (qtp759257841-18439) [n:127.0.0.1:45567_solr c:hdfsbackuprestore_testfailure s:shard2 r:core_node9 x:hdfsbackuprestore_testfailure_shard2_replica_n6 ] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1303272 INFO  (qtp759257841-18439) [n:127.0.0.1:45567_solr c:hdfsbackuprestore_testfailure s:shard2 r:core_node9 x:hdfsbackuprestore_testfailure_shard2_replica_n6 ] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1303274 INFO  (qtp1111921956-18440) [n:127.0.0.1:40407_solr c:hdfsbackuprestore_testfailure s:shard1 r:core_node7 x:hdfsbackuprestore_testfailure_shard1_replica_p4 ] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1303274 INFO  (qtp1111921956-18440) [n:127.0.0.1:40407_solr c:hdfsbackuprestore_testfailure s:shard1 r:core_node7 x:hdfsbackuprestore_testfailure_shard1_replica_p4 ] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1303275 INFO  (qtp759257841-18436) [n:127.0.0.1:45567_solr c:hdfsbackuprestore_testfailure s:shard1 r:core_node5 x:hdfsbackuprestore_testfailure_shard1_replica_t2 ] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1303275 INFO  (qtp1111921956-18433) [n:127.0.0.1:40407_solr c:hdfsbackuprestore_testfailure s:shard2 r:core_node11 x:hdfsbackuprestore_testfailure_shard2_replica_t8 ] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with zno

[...truncated too long message...]

.s.c.ZkTestServer connecting to 127.0.0.1 32921
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.servlet.TestRequestRateLimiter_707BAB5045BCF7AA-001
   [junit4]   2> Aug 08, 2020 6:01:38 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene86): {}, docValues:{}, maxPointsInLeafNode=618, maxMBSortInHeap=7.936571126338736, sim=Asserting(RandomSimilarity(queryNorm=true): {}), locale=mfe, timezone=America/Resolute
   [junit4]   2> NOTE: Linux 4.15.0-108-generic amd64/Oracle Corporation 11.0.6 (64-bit)/cpus=4,threads=1,free=264884272,total=414187520
   [junit4]   2> NOTE: All tests run in this JVM: [XmlUpdateRequestHandlerTest, DateMathParserTest, AddSchemaFieldsUpdateProcessorFactoryTest, RestartWhileUpdatingTest, CoreSorterTest, TestTolerantUpdateProcessorRandomCloud, TestManagedSchema, UninvertDocValuesMergePolicyTest, DistributedFacetExistsSmallTest, TestClassicSimilarityFactory, TestSolrJ, TestCloudJSONFacetJoinDomain, TestSolrConfigHandlerCloud, TestEmbeddedSolrServerSchemaAPI, TestRuleBasedAuthorizationWithKerberos, TestSolrCoreSnapshots, DistributedFacetPivotSmallAdvancedTest, AssignBackwardCompatibilityTest, SuggesterTSTTest, TestQuerySenderNoQuery, TestXIncludeConfig, OutOfBoxZkACLAndCredentialsProvidersTest, TestChildDocTransformerHierarchy, TestDownShardTolerantSearch, TestOnReconnectListenerSupport, AlternateDirectoryTest, HLLUtilTest, SimpleCollectionCreateDeleteTest, SolrCLIZkUtilsTest, TestRandomFlRTGCloud, TestStressCloudBlindAtomicUpdates, RAMDirectoryFactoryTest, AtomicUpdatesTest, BJQParserTest, HdfsNNFailoverTest, TestSimpleTextCodec, CloudMLTQParserTest, BooleanFieldTest, TestUnifiedSolrHighlighter, TestPullReplicaErrorHandling, TestCloudPivotFacet, AuditLoggerPluginTest, ZkStateReaderTest, ShardRoutingTest, TestPseudoReturnFields, ChaosMonkeyNothingIsSafeTest, TestTestInjection, EchoParamsTest, TestTlogReplayVsRecovery, TestBulkSchemaConcurrent, TestLegacyTerms, TestAnalyzeInfixSuggestions, TestConfig, StressHdfsTest, LargeFieldTest, TestExportTool, CurrencyRangeFacetCloudTest, MoveReplicaHDFSTest, DefaultValueUpdateProcessorTest, MigrateRouteKeyTest, HealthCheckHandlerTest, ChaosMonkeySafeLeaderWithPullReplicasTest, TestOrdValues, SolrSlf4jReporterTest, BigEndianAscendingWordDeserializerTest, TestManagedResourceStorage, TestRequestForwarding, LeaderElectionContextKeyTest, RankFieldTest, TestCollectionsAPIViaSolrCloudCluster, CdcrWithNodesRestartsTest, SolrIndexSearcherTest, SOLR749Test, HdfsTlogReplayBufferedWhileIndexingTest, SolrIndexSplitterTest, TestHashQParserPlugin, SolrMetricManagerTest, DistributedFacetSimpleRefinementLongTailTest, TestReplicationHandlerBackup, QueryEqualityTest, TestFieldCache, TestComplexPhraseLeadingWildcard, TestJsonFacetErrors, SecurityConfHandlerTest, NumericFieldsTest, TestCoreBackup, CollectionsAPISolrJTest, TestSubQueryTransformerCrossCore, TestExpandComponent, TestNumericRangeQuery64, TriLevelCompositeIdRoutingTest, TestLegacyNumericRangeQueryBuilder, TestCustomSort, TolerantUpdateProcessorTest, SystemInfoHandlerTest, TestTextField, TestAddFieldRealTimeGet, TestExtendedDismaxParser, BasicAuthStandaloneTest, TestCloudRecovery2, ConnectionManagerTest, TestOmitPositions, TestHighFrequencyDictionaryFactory, TestConfigSetProperties, CursorPagingTest, SuggestComponentContextFilterQueryTest, TestRecoveryHdfs, TestScoreJoinQPScore, TestValueSourceCache, BlockDirectoryTest, AsyncCallRequestStatusResponseTest, TestShortCircuitedRequests, DocumentBuilderTest, FileUtilsTest, SpellCheckCollatorWithCollapseTest, TestSurroundQueryParser, SortByFunctionTest, TestDocBasedVersionConstraints, CloudExitableDirectoryReaderTest, TestIBSimilarityFactory, ResponseBuilderTest, HdfsRestartWhileUpdatingTest, TestSchemaManager, FileBasedSpellCheckerTest, LeaderVoteWaitTimeoutTest, TestPushWriter, JSONWriterTest, TestManagedSynonymGraphFilterFactory, TestTrie, JvmMetricsTest, DistributedMLTComponentTest, PreAnalyzedFieldManagedSchemaCloudTest, TestFieldTypeResource, DistributedTermsComponentTest, LoggingHandlerTest, NoCacheHeaderTest, HdfsDirectoryFactoryTest, TestCSVResponseWriter, TestCollationField, TestFacetMethods, CoreAdminCreateDiscoverTest, AnalysisAfterCoreReloadTest, BasicFunctionalityTest, ConvertedLegacyTest, SampleTest, SolrInfoBeanTest, SolrTestCaseJ4Test, TestCrossCoreJoin, TestCursorMarkWithoutUniqueKey, TestDistributedGrouping, TestDistributedMissingSort, TestDistributedSearch, TestSimpleTrackingShardHandler, TestSolrTestCaseJ4, TestCharFilters, TestLuceneMatchVersion, TestWordDelimiterFilterFactory, TestEmbeddedSolrServerAdminHandler, TestEmbeddedSolrServerConstructors, ActionThrottleTest, AddReplicaTest, AliasIntegrationTest, DeleteLastCustomShardedReplicaTest, DeleteNodeTest, DeleteReplicaTest, DistribDocExpirationUpdateProcessorTest, ForceLeaderWithTlogReplicasTest, HttpPartitionOnCommitTest, HttpPartitionWithTlogReplicasTest, LeaderFailoverAfterPartitionTest, LeaderTragicEventTest, MultiThreadedOCPTest, OverseerRolesTest, OverseerTest, PeerSyncReplicationTest, RemoteQueryErrorTest, ReplaceNodeTest, ReplicationFactorTest, SyncSliceTest, TestCloudDeleteByQuery, TestCloudInspectUtil, TestCloudPhrasesIdentificationComponent, TestConfigSetsAPI, TestConfigSetsAPIExclusivity, TestConfigSetsAPIZkFailure, TestDeleteCollectionOnDownNodes, TestDistribDocBasedVersion, TestExactStatsCacheCloud, TestLRUStatsCacheCloud, TestLeaderElectionWithEmptyReplica, TestLockTree, TestSolrCloudWithDelegationTokens, TestSolrCloudWithKerberosAlt, TestSolrCloudWithSecureImpersonation, ZkCLITest, ZkSolrClientTest, CollectionsAPIDistributedZkTest, CustomCollectionTest, HdfsCollectionsAPIDistributedZkTest, TestReplicaProperties, TestRequestStatusCollectionAPI, BaseCdcrDistributedZkTest, CdcrBidirectionalTest, CdcrBootstrapTest, CdcrOpsAndBoundariesTest, HdfsChaosMonkeyNothingIsSafeTest, HdfsChaosMonkeySafeLeaderTest, RuleEngineTest, RulesTest, ZkDynamicConfigTest, ByteBuffersDirectoryFactoryTest, CachingDirectoryFactoryTest, ConfigureRecoveryStrategyTest, TestBackupRepositoryFactory, TestBadConfig, TestCodecSupport, TestCoreDiscovery, TestCorePropertiesReload, TestCustomStream, TestDirectoryFactory, TestImplicitCoreProperties, TestInfoStreamLogging, TestInitParams, TestJmxIntegration, TestLazyCores, TestMergePolicyConfig, TestNRTOpen, TestQuerySenderListener, TestSolrDeletionPolicy2, TestSolrIndexConfig, TestSolrXml, CSVRequestHandlerTest, DocumentAnalysisRequestHandlerTest, FieldAnalysisRequestHandlerTest, JsonLoaderTest, MoreLikeThisHandlerTest, PingRequestHandlerTest, RequestLoggingTest, SearchHandlerTest, TestBlobHandler, TestCSVLoader, TestConfigReload, TestContainerPlugin, TestRestoreCore, TestSQLHandler, TestSQLHandlerNonCloud, TestSystemCollAutoCreate, V2ApiIntegrationTest, V2StandaloneTest, CoreAdminRequestStatusTest, CoreMergeIndexesAdminHandlerTest, DaemonStreamApiTest, MetricsHandlerTest, MetricsHistoryHandlerTest, PropertiesRequestHandlerTest, TestConfigsApi, TestCoreAdminApis, ThreadDumpHandlerTest, ZookeeperReadAPITest, ZookeeperStatusHandlerTest, BadComponentTest, CloudReplicaSourceTest, CustomHighlightComponentTest, CustomTermsComponentTest, DebugComponentTest, DistributedDebugComponentTest, DistributedExpandComponentTest, DistributedQueryElevationComponentTest, DistributedSpellCheckComponentTest, DistributedSuggestComponentTest, MoreLikeThisComponentTest, PhrasesIdentificationComponentTest, QueryElevationComponentTest, ShardsWhitelistTest, SpellCheckComponentTest, StatsComponentTest, TermVectorComponentTest, TermsComponentTest, TestDistributedStatsComponentCardinality, TestTrackingShardHandlerFactory, TestExportWriter, JavabinLoaderTest, EmbeddedSolrNoSerializeTest, RandomizedTaggerTest, Tagger2Test, TaggerTest, TaggingAttributeTest, XmlInterpolationTest, FastVectorHighlighterTest, HighlighterConfigTest, HighlighterMaxOffsetTest, HighlighterTest, HighlighterWithoutStoredIdTest, TestPostingsSolrHighlighter, TestLegacyField, TestLegacyFieldReuse, TestLegacyNumericUtils, SolrGraphiteReporterTest, SolrJmxReporterCloudTest, SolrJmxReporterTest, SolrRrdBackendFactoryTest, TestPackages, RegexBytesRefFilterTest, SimpleFacetsTest, SubstringBytesRefFilterTest, TestRemoteStreaming, TestStreamBody, TestUnInvertedFieldException, TestWriterPerf, TestMacroExpander, TestMacros, TestGraphMLResponseWriter, TestJavabinTupleStreamParser, TestPHPSerializedResponseWriter, TestRetrieveFieldsOptimizer, TestSolrQueryResponse, TestChildDocTransformer, TestCopyFieldCollectionResource, TestDynamicFieldCollectionResource, TestDynamicFieldResource, TestFieldCollectionResource, TestFieldResource, TestFieldTypeCollectionResource, TestSchemaSimilarityResource, TestSchemaVersionResource, TestSerializedLuceneMatchVersion, TestUniqueKeyFieldResource, TestManagedStopFilterFactory, TestManagedSynonymFilterFactory, CurrencyFieldTypeTest, DateFieldTest, DocValuesMissingTest, EnumFieldTest, ExternalFileFieldSortTest, IndexSchemaRuntimeFieldTest, IndexSchemaTest, ManagedSchemaRoundRobinCloudTest, NotRequiredUniqueKeyTest, SchemaApiFailureTest, SpatialRPTFieldTypeTest, TestFoldingMultitermQuery, TestGraphTermsQParserPlugin, TestMissingGroups, TestOverriddenPrefixQueryForCustomFieldType, TestPayloadScoreQParserPlugin, TestRTGBase, TestRangeQuery, TestStandardQParsers, TestStressLucene, TestStressUserVersions, TestStressVersions, TestCloudNestedDocsSort, TestScoreJoinQPNoScore, TestDistribIDF, TestExactStatsCache, AuditLoggerIntegrationTest, MultiDestinationAuditLoggerTest, PKIAuthenticationIntegrationTest, SolrLogAuditLoggerPluginTest, TestAuthorizationFramework, DirectSolrConnectionTest, TestRequestRateLimiter]
   [junit4] Completed [776/864 (3!)] on J0 in 3.50s, 2 tests, 2 failures <<< FAILURES!

[...truncated 52153 lines...]

[JENKINS] Lucene » Lucene-Solr-NightlyTests-master - Build # 6 - Still Unstable!

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://ci-builds.apache.org/job/Lucene/job/Lucene-Solr-NightlyTests-master/6/

2 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.schema.TestBulkSchemaConcurrent

Error Message:
ObjectTracker found 3 object(s) that were not released!!! [NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.NRTCachingDirectory  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:354)  at org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:406)  at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:785)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1005)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:925)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1357)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1243)  at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:95)  at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:367)  at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:397)  at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:181)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:212)  at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:854)  at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:818)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:566)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:437)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:351)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:166)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:545)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1610)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1300)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:485)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1580)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1215)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)  at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:717)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)  at org.eclipse.jetty.server.Server.handle(Server.java:500)  at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383)  at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:547)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375)  at org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:335)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)  at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)  at java.base/java.lang.Thread.run(Thread.java:834)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.NRTCachingDirectory  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:354)  at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:98)  at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:817)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1005)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:925)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1357)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1243)  at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:95)  at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:367)  at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:397)  at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:181)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:212)  at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:854)  at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:818)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:566)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:437)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:351)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:166)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:545)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1610)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1300)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:485)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1580)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1215)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)  at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:717)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)  at org.eclipse.jetty.server.Server.handle(Server.java:500)  at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383)  at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:547)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375)  at org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:335)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)  at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)  at java.base/java.lang.Thread.run(Thread.java:834)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.NRTCachingDirectory  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:354)  at org.apache.solr.core.SolrCore.initSnapshotMetaDataManager(SolrCore.java:557)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1001)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:925)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1357)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1243)  at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:95)  at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:367)  at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:397)  at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:181)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:212)  at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:854)  at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:818)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:566)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:437)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:351)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:166)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:545)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1610)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1300)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:485)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1580)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1215)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)  at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:717)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)  at org.eclipse.jetty.server.Server.handle(Server.java:500)  at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383)  at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:547)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375)  at org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:335)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)  at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)  at java.base/java.lang.Thread.run(Thread.java:834)   expected null, but was:<ObjectTracker found 3 object(s) that were not released!!! [NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.NRTCachingDirectory  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:354)  at org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:406)  at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:785)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1005)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:925)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1357)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1243)  at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:95)  at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:367)  at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:397)  at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:181)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:212)  at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:854)  at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:818)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:566)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:437)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:351)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:166)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:545)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1610)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1300)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:485)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1580)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1215)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)  at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:717)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)  at org.eclipse.jetty.server.Server.handle(Server.java:500)  at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383)  at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:547)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375)  at org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:335)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)  at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)  at java.base/java.lang.Thread.run(Thread.java:834)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.NRTCachingDirectory  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:354)  at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:98)  at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:817)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1005)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:925)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1357)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1243)  at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:95)  at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:367)  at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:397)  at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:181)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:212)  at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:854)  at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:818)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:566)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:437)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:351)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:166)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:545)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1610)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1300)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:485)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1580)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1215)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)  at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:717)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)  at org.eclipse.jetty.server.Server.handle(Server.java:500)  at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383)  at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:547)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375)  at org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:335)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)  at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)  at java.base/java.lang.Thread.run(Thread.java:834)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.NRTCachingDirectory  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:354)  at org.apache.solr.core.SolrCore.initSnapshotMetaDataManager(SolrCore.java:557)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1001)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:925)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1357)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1243)  at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:95)  at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:367)  at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:397)  at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:181)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:212)  at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:854)  at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:818)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:566)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:437)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:351)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:166)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:545)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1610)  at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1300)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:485)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1580)  at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1215)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)  at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:717)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)  at org.eclipse.jetty.server.Server.handle(Server.java:500)  at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383)  at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:547)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375)  at org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:335)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)  at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)  at java.base/java.lang.Thread.run(Thread.java:834)  >

Stack Trace:
java.lang.AssertionError: ObjectTracker found 3 object(s) that were not released!!! [NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.NRTCachingDirectory
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
	at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:354)
	at org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:406)
	at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:785)
	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1005)
	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:925)
	at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1357)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1243)
	at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:95)
	at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:367)
	at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:397)
	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:181)
	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:212)
	at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:854)
	at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:818)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:566)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:437)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:351)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:166)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:545)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1610)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1300)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:485)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1580)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1215)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:717)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at org.eclipse.jetty.server.Server.handle(Server.java:500)
	at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383)
	at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:547)
	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375)
	at org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:335)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)
	at java.base/java.lang.Thread.run(Thread.java:834)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.NRTCachingDirectory
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
	at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:354)
	at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:98)
	at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:817)
	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1005)
	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:925)
	at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1357)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1243)
	at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:95)
	at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:367)
	at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:397)
	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:181)
	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:212)
	at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:854)
	at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:818)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:566)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:437)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:351)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:166)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:545)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1610)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1300)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:485)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1580)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1215)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:717)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at org.eclipse.jetty.server.Server.handle(Server.java:500)
	at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383)
	at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:547)
	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375)
	at org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:335)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)
	at java.base/java.lang.Thread.run(Thread.java:834)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.NRTCachingDirectory
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
	at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:354)
	at org.apache.solr.core.SolrCore.initSnapshotMetaDataManager(SolrCore.java:557)
	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1001)
	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:925)
	at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1357)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1243)
	at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:95)
	at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:367)
	at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:397)
	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:181)
	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:212)
	at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:854)
	at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:818)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:566)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:437)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:351)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:166)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:545)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1610)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1300)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:485)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1580)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1215)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:717)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at org.eclipse.jetty.server.Server.handle(Server.java:500)
	at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383)
	at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:547)
	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375)
	at org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:335)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)
	at java.base/java.lang.Thread.run(Thread.java:834)

 expected null, but was:<ObjectTracker found 3 object(s) that were not released!!! [NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.NRTCachingDirectory
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
	at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:354)
	at org.apache.solr.core.SolrCore.getNewIndexDir(SolrCore.java:406)
	at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:785)
	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1005)
	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:925)
	at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1357)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1243)
	at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:95)
	at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:367)
	at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:397)
	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:181)
	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:212)
	at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:854)
	at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:818)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:566)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:437)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:351)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:166)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:545)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1610)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1300)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:485)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1580)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1215)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:717)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at org.eclipse.jetty.server.Server.handle(Server.java:500)
	at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383)
	at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:547)
	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375)
	at org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:335)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)
	at java.base/java.lang.Thread.run(Thread.java:834)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.NRTCachingDirectory
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
	at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:354)
	at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:98)
	at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:817)
	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1005)
	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:925)
	at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1357)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1243)
	at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:95)
	at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:367)
	at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:397)
	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:181)
	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:212)
	at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:854)
	at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:818)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:566)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:437)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:351)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:166)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:545)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1610)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1300)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:485)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1580)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1215)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:717)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at org.eclipse.jetty.server.Server.handle(Server.java:500)
	at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383)
	at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:547)
	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375)
	at org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:335)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)
	at java.base/java.lang.Thread.run(Thread.java:834)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.NRTCachingDirectory
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
	at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:354)
	at org.apache.solr.core.SolrCore.initSnapshotMetaDataManager(SolrCore.java:557)
	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1001)
	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:925)
	at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1357)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1243)
	at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:95)
	at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:367)
	at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:397)
	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:181)
	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:212)
	at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:854)
	at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:818)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:566)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:437)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:351)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:166)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:545)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1610)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1300)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:485)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1580)
	at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1215)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:717)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
	at org.eclipse.jetty.server.Server.handle(Server.java:500)
	at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383)
	at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:547)
	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375)
	at org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:335)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)
	at java.base/java.lang.Thread.run(Thread.java:834)

>
	at __randomizedtesting.SeedInfo.seed([1F01E002FCAD9B32]:0)
	at org.junit.Assert.fail(Assert.java:88)
	at org.junit.Assert.failNotNull(Assert.java:755)
	at org.junit.Assert.assertNull(Assert.java:737)
	at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:327)
	at jdk.internal.reflect.GeneratedMethodAccessor224.invoke(Unknown Source)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:905)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:834)


FAILED:  org.apache.solr.search.TestReRankQParserPlugin.testMinExactCount

Error Message:
Exception during query

Stack Trace:
java.lang.RuntimeException: Exception during query
	at __randomizedtesting.SeedInfo.seed([1F01E002FCAD9B32:6A81DA9A13706545]:0)
	at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:1000)
	at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:960)
	at org.apache.solr.search.TestReRankQParserPlugin.testMinExactCount(TestReRankQParserPlugin.java:675)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:834)
Caused by: java.lang.RuntimeException: REQUEST FAILED: xpath=//result/doc[1]/str[@name='id'][.='0']
	xml response was: <?xml version="1.0" encoding="UTF-8"?>
<response>
<lst name="responseHeader"><int name="status">0</int><int name="QTime">0</int></lst><result name="response" numFound="200" start="0" numFoundExact="true"><doc><str name="id">108</str></doc><doc><str name="id">109</str></doc><doc><str name="id">110</str></doc><doc><str name="id">111</str></doc><doc><str name="id">112</str></doc><doc><str name="id">113</str></doc><doc><str name="id">126</str></doc><doc><str name="id">127</str></doc><doc><str name="id">128</str></doc><doc><str name="id">129</str></doc></result>
</response>

	request was:q=field_t:0&start=0&rows=10&fl=id
	at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:993)
	... 41 more




Build Log:
[...truncated 14513 lines...]
   [junit4] Suite: org.apache.solr.schema.TestBulkSchemaConcurrent
   [junit4]   2> 4835872 INFO  (SUITE-TestBulkSchemaConcurrent-seed#[1F01E002FCAD9B32]-worker) [     ] o.a.s.SolrTestCase Setting 'solr.default.confdir' system property to test-framework derived value of '/home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/server/solr/configsets/_default/conf'
   [junit4]   2> 4835873 INFO  (SUITE-TestBulkSchemaConcurrent-seed#[1F01E002FCAD9B32]-worker) [     ] o.a.s.SolrTestCaseJ4 Created dataDir: /home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.schema.TestBulkSchemaConcurrent_1F01E002FCAD9B32-001/data-dir-254-001
   [junit4]   2> 4835873 WARN  (SUITE-TestBulkSchemaConcurrent-seed#[1F01E002FCAD9B32]-worker) [     ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=2 numCloses=2
   [junit4]   2> 4835873 INFO  (SUITE-TestBulkSchemaConcurrent-seed#[1F01E002FCAD9B32]-worker) [     ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 4835874 INFO  (SUITE-TestBulkSchemaConcurrent-seed#[1F01E002FCAD9B32]-worker) [     ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason="", ssl=0.0/0.0, value=0.0/0.0, clientAuth=0.0/0.0)
   [junit4]   2> 4835874 INFO  (SUITE-TestBulkSchemaConcurrent-seed#[1F01E002FCAD9B32]-worker) [     ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 4835874 INFO  (SUITE-TestBulkSchemaConcurrent-seed#[1F01E002FCAD9B32]-worker) [     ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /rqj/xd
   [junit4]   2> 4835877 INFO  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [     ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 4835878 INFO  (ZkTestServer Run Thread) [     ] o.a.s.c.ZkTestServer client port: 0.0.0.0/0.0.0.0:0
   [junit4]   2> 4835878 INFO  (ZkTestServer Run Thread) [     ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 4835879 WARN  (ZkTestServer Run Thread) [     ] o.a.z.s.ServerCnxnFactory maxCnxns is not configured, using default value 0.
   [junit4]   2> 4835978 INFO  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [     ] o.a.s.c.ZkTestServer start zk server on port: 41665
   [junit4]   2> 4835978 INFO  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [     ] o.a.s.c.ZkTestServer waitForServerUp: 127.0.0.1:41665
   [junit4]   2> 4835978 INFO  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [     ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:41665
   [junit4]   2> 4835978 INFO  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [     ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 41665
   [junit4]   2> 4835980 INFO  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 4835982 INFO  (zkConnectionManagerCallback-31908-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4835982 INFO  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 4835992 INFO  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 4835993 INFO  (zkConnectionManagerCallback-31910-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4835993 INFO  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 4835994 INFO  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-managed-schema.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 4835996 INFO  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 4835998 INFO  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 4835999 INFO  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 4835999 INFO  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 4836000 INFO  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 4836001 INFO  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 4836002 INFO  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 4836003 INFO  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 4836004 INFO  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 4836005 INFO  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [     ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 4836006 INFO  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [     ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
   [junit4]   2> 4836107 INFO  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
   [junit4]   2> 4836107 INFO  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 2 ...
   [junit4]   2> 4836107 INFO  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [     ] o.e.j.s.Server jetty-9.4.27.v20200227; built: 2020-02-27T18:37:21.340Z; git: a304fd9f351f337e7c0e2a7c28878dd536149c6c; jvm 11.0.6+8-LTS
   [junit4]   2> 4836108 INFO  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 4836108 INFO  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 4836108 INFO  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [     ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 4836112 INFO  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@798dab86{/rqj/xd,null,AVAILABLE}
   [junit4]   2> 4836113 INFO  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [     ] o.e.j.s.AbstractConnector Started ServerConnector@42c3c891{HTTP/1.1, (http/1.1, h2c)}{127.0.0.1:42087}
   [junit4]   2> 4836113 INFO  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [     ] o.e.j.s.Server Started @4836153ms
   [junit4]   2> 4836113 INFO  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/rqj/xd, solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.schema.TestBulkSchemaConcurrent_1F01E002FCAD9B32-001/tempDir-001/control/data, hostPort=42087, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.schema.TestBulkSchemaConcurrent_1F01E002FCAD9B32-001/control-001/cores}
   [junit4]   2> 4836113 ERROR (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [     ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 4836113 INFO  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [     ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 4836113 INFO  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [     ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 9.0.0
   [junit4]   2> 4836113 INFO  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [     ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 4836113 INFO  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [     ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: /home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr
   [junit4]   2> 4836113 INFO  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [     ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2020-08-13T02:48:32.944719Z
   [junit4]   2> 4836114 INFO  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 4836116 INFO  (zkConnectionManagerCallback-31912-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4836116 INFO  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 4836218 INFO  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [     ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 4836218 INFO  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [     ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.schema.TestBulkSchemaConcurrent_1F01E002FCAD9B32-001/control-001/solr.xml
   [junit4]   2> 4836222 INFO  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [     ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@33e74307, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 4836468 INFO  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [     ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
   [junit4]   2> 4836469 WARN  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@372a5118[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 4836469 WARN  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@372a5118[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 4836472 WARN  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@5447ec8b[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 4836472 WARN  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@5447ec8b[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 4836474 INFO  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [     ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:41665/solr
   [junit4]   2> 4836476 INFO  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 4836478 INFO  (zkConnectionManagerCallback-31922-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4836478 INFO  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 4836581 INFO  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [n:127.0.0.1:42087_rqj%2Fxd     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 4836582 INFO  (zkConnectionManagerCallback-31924-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4836582 INFO  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [n:127.0.0.1:42087_rqj%2Fxd     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 4836657 INFO  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [n:127.0.0.1:42087_rqj%2Fxd     ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:42087_rqj%2Fxd
   [junit4]   2> 4836658 INFO  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [n:127.0.0.1:42087_rqj%2Fxd     ] o.a.s.c.Overseer Overseer (id=72302333454909444-127.0.0.1:42087_rqj%2Fxd-n_0000000000) starting
   [junit4]   2> 4836662 INFO  (OverseerStateUpdate-72302333454909444-127.0.0.1:42087_rqj%2Fxd-n_0000000000) [n:127.0.0.1:42087_rqj%2Fxd     ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:42087_rqj%2Fxd
   [junit4]   2> 4836662 INFO  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [n:127.0.0.1:42087_rqj%2Fxd     ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:42087_rqj%2Fxd
   [junit4]   2> 4836663 INFO  (OverseerStateUpdate-72302333454909444-127.0.0.1:42087_rqj%2Fxd-n_0000000000) [n:127.0.0.1:42087_rqj%2Fxd     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 4836665 INFO  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [n:127.0.0.1:42087_rqj%2Fxd     ] o.a.s.p.PackageLoader /packages.json updated to version -1
   [junit4]   2> 4836665 WARN  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [n:127.0.0.1:42087_rqj%2Fxd     ] o.a.s.c.CoreContainer Not all security plugins configured!  authentication=disabled authorization=disabled.  Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external.  See https://s.apache.org/solrsecurity for more info
   [junit4]   2> 4836679 INFO  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [n:127.0.0.1:42087_rqj%2Fxd     ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 4836697 INFO  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [n:127.0.0.1:42087_rqj%2Fxd     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@33e74307
   [junit4]   2> 4836704 INFO  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [n:127.0.0.1:42087_rqj%2Fxd     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@33e74307
   [junit4]   2> 4836704 INFO  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [n:127.0.0.1:42087_rqj%2Fxd     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@33e74307
   [junit4]   2> 4836705 INFO  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [n:127.0.0.1:42087_rqj%2Fxd     ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.schema.TestBulkSchemaConcurrent_1F01E002FCAD9B32-001/control-001/cores
   [junit4]   2> 4836713 INFO  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 4836714 INFO  (zkConnectionManagerCallback-31939-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4836714 INFO  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 4836716 INFO  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 4836717 INFO  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [     ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:41665/solr ready
   [junit4]   2> 4836721 INFO  (qtp2053050546-159710) [n:127.0.0.1:42087_rqj%2Fxd     ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:42087_rqj%252Fxd&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 4836724 INFO  (OverseerThreadFactory-31929-thread-1-processing-n:127.0.0.1:42087_rqj%2Fxd) [n:127.0.0.1:42087_rqj%2Fxd     ] o.a.s.c.a.c.CreateCollectionCmd Create collection control_collection
   [junit4]   2> 4836831 INFO  (OverseerStateUpdate-72302333454909444-127.0.0.1:42087_rqj%2Fxd-n_0000000000) [n:127.0.0.1:42087_rqj%2Fxd     ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"control_collection",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"control_collection_shard1_replica_n1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:42087/rqj/xd",
   [junit4]   2>   "node_name":"127.0.0.1:42087_rqj%2Fxd",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 4837035 INFO  (qtp2053050546-159713) [n:127.0.0.1:42087_rqj%2Fxd    x:control_collection_shard1_replica_n1 ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node2&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 4837036 INFO  (qtp2053050546-159713) [n:127.0.0.1:42087_rqj%2Fxd    x:control_collection_shard1_replica_n1 ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 4837044 INFO  (qtp2053050546-159713) [n:127.0.0.1:42087_rqj%2Fxd c:control_collection s:shard1 r:core_node2 x:control_collection_shard1_replica_n1 ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.0.0
   [junit4]   2> 4837051 INFO  (qtp2053050546-159713) [n:127.0.0.1:42087_rqj%2Fxd c:control_collection s:shard1 r:core_node2 x:control_collection_shard1_replica_n1 ] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from configset conf1, trusted=true
   [junit4]   2> 4837051 INFO  (qtp2053050546-159713) [n:127.0.0.1:42087_rqj%2Fxd c:control_collection s:shard1 r:core_node2 x:control_collection_shard1_replica_n1 ] o.a.s.s.ManagedIndexSchemaFactory The schema is configured as managed, but managed schema resource managed-schema not found - loading non-managed schema schema.xml instead
   [junit4]   2> 4837057 INFO  (qtp2053050546-159713) [n:127.0.0.1:42087_rqj%2Fxd c:control_collection s:shard1 r:core_node2 x:control_collection_shard1_replica_n1 ] o.a.s.s.IndexSchema Schema name=test
   [junit4]   2> 4837165 INFO  (qtp2053050546-159713) [n:127.0.0.1:42087_rqj%2Fxd c:control_collection s:shard1 r:core_node2 x:control_collection_shard1_replica_n1 ] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
   [junit4]   2> 4837192 INFO  (qtp2053050546-159713) [n:127.0.0.1:42087_rqj%2Fxd c:control_collection s:shard1 r:core_node2 x:control_collection_shard1_replica_n1 ] o.a.s.s.ManagedIndexSchema Created and persisted managed schema znode at /configs/conf1/managed-schema
   [junit4]   2> 4837195 INFO  (qtp2053050546-159713) [n:127.0.0.1:42087_rqj%2Fxd c:control_collection s:shard1 r:core_node2 x:control_collection_shard1_replica_n1 ] o.a.s.s.ManagedIndexSchemaFactory After upgrading to managed schema in ZooKeeper, renamed the non-managed schema /configs/conf1/schema.xml to /configs/conf1/schema.xml.bak
   [junit4]   2> 4837196 INFO  (qtp2053050546-159713) [n:127.0.0.1:42087_rqj%2Fxd c:control_collection s:shard1 r:core_node2 x:control_collection_shard1_replica_n1 ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@33e74307
   [junit4]   2> 4837196 INFO  (qtp2053050546-159713) [n:127.0.0.1:42087_rqj%2Fxd c:control_collection s:shard1 r:core_node2 x:control_collection_shard1_replica_n1 ] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.schema.TestBulkSchemaConcurrent_1F01E002FCAD9B32-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.schema.TestBulkSchemaConcurrent_1F01E002FCAD9B32-001/control-001/cores/control_collection_shard1_replica_n1/data/]
   [junit4]   2> 4837199 INFO  (qtp2053050546-159713) [n:127.0.0.1:42087_rqj%2Fxd c:control_collection s:shard1 r:core_node2 x:control_collection_shard1_replica_n1 ] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=43, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.13394919667009264]
   [junit4]   2> 4837214 WARN  (qtp2053050546-159713) [n:127.0.0.1:42087_rqj%2Fxd c:control_collection s:shard1 r:core_node2 x:control_collection_shard1_replica_n1 ] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, registerPath=/solr,/v2, class=DumpRequestHandler},args = {defaults={a=A, b=B}}}
   [junit4]   2> 4837265 INFO  (qtp2053050546-159713) [n:127.0.0.1:42087_rqj%2Fxd c:control_collection s:shard1 r:core_node2 x:control_collection_shard1_replica_n1 ] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 4837265 INFO  (qtp2053050546-159713) [n:127.0.0.1:42087_rqj%2Fxd c:control_collection s:shard1 r:core_node2 x:control_collection_shard1_replica_n1 ] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 4837266 INFO  (qtp2053050546-159713) [n:127.0.0.1:42087_rqj%2Fxd c:control_collection s:shard1 r:core_node2 x:control_collection_shard1_replica_n1 ] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 4837266 INFO  (qtp2053050546-159713) [n:127.0.0.1:42087_rqj%2Fxd c:control_collection s:shard1 r:core_node2 x:control_collection_shard1_replica_n1 ] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 4837267 INFO  (qtp2053050546-159713) [n:127.0.0.1:42087_rqj%2Fxd c:control_collection s:shard1 r:core_node2 x:control_collection_shard1_replica_n1 ] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=8, maxMergeAtOnceExplicit=5, maxMergedSegmentMB=61.7734375, floorSegmentMB=0.693359375, forceMergeDeletesPctAllowed=27.81931911644941, segmentsPerTier=47.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5059772712499399, deletesPctAllowed=43.15504390061368
   [junit4]   2> 4837270 INFO  (qtp2053050546-159713) [n:127.0.0.1:42087_rqj%2Fxd c:control_collection s:shard1 r:core_node2 x:control_collection_shard1_replica_n1 ] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 4837270 INFO  (qtp2053050546-159713) [n:127.0.0.1:42087_rqj%2Fxd c:control_collection s:shard1 r:core_node2 x:control_collection_shard1_replica_n1 ] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 4837270 INFO  (qtp2053050546-159713) [n:127.0.0.1:42087_rqj%2Fxd c:control_collection s:shard1 r:core_node2 x:control_collection_shard1_replica_n1 ] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/conf1/managed-schema
   [junit4]   2> 4837271 INFO  (qtp2053050546-159713) [n:127.0.0.1:42087_rqj%2Fxd c:control_collection s:shard1 r:core_node2 x:control_collection_shard1_replica_n1 ] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
   [junit4]   2> 4837271 INFO  (qtp2053050546-159713) [n:127.0.0.1:42087_rqj%2Fxd c:control_collection s:shard1 r:core_node2 x:control_collection_shard1_replica_n1 ] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 ms
   [junit4]   2> 4837271 INFO  (qtp2053050546-159713) [n:127.0.0.1:42087_rqj%2Fxd c:control_collection s:shard1 r:core_node2 x:control_collection_shard1_replica_n1 ] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1674876723241418752
   [junit4]   2> 4837274 INFO  (searcherExecutor-31941-thread-1-processing-n:127.0.0.1:42087_rqj%2Fxd x:control_collection_shard1_replica_n1 c:control_collection s:shard1 r:core_node2) [n:127.0.0.1:42087_rqj%2Fxd c:control_collection s:shard1 r:core_node2 x:control_collection_shard1_replica_n1 ] o.a.s.c.SolrCore [control_collection_shard1_replica_n1]  Registered new searcher autowarm time: 0 ms
   [junit4]   2> 4837278 INFO  (qtp2053050546-159713) [n:127.0.0.1:42087_rqj%2Fxd c:control_collection s:shard1 r:core_node2 x:control_collection_shard1_replica_n1 ] o.a.s.c.ZkShardTerms Successful update of terms at /collections/control_collection/terms/shard1 to Terms{values={core_node2=0}, version=0}
   [junit4]   2> 4837278 INFO  (qtp2053050546-159713) [n:127.0.0.1:42087_rqj%2Fxd c:control_collection s:shard1 r:core_node2 x:control_collection_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/control_collection/leaders/shard1
   [junit4]   2> 4837281 INFO  (qtp2053050546-159713) [n:127.0.0.1:42087_rqj%2Fxd c:control_collection s:shard1 r:core_node2 x:control_collection_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 4837281 INFO  (qtp2053050546-159713) [n:127.0.0.1:42087_rqj%2Fxd c:control_collection s:shard1 r:core_node2 x:control_collection_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 4837281 INFO  (qtp2053050546-159713) [n:127.0.0.1:42087_rqj%2Fxd c:control_collection s:shard1 r:core_node2 x:control_collection_shard1_replica_n1 ] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:42087/rqj/xd/control_collection_shard1_replica_n1/
   [junit4]   2> 4837281 INFO  (qtp2053050546-159713) [n:127.0.0.1:42087_rqj%2Fxd c:control_collection s:shard1 r:core_node2 x:control_collection_shard1_replica_n1 ] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 4837281 INFO  (qtp2053050546-159713) [n:127.0.0.1:42087_rqj%2Fxd c:control_collection s:shard1 r:core_node2 x:control_collection_shard1_replica_n1 ] o.a.s.c.SyncStrategy http://127.0.0.1:42087/rqj/xd/control_collection_shard1_replica_n1/ has no replicas
   [junit4]   2> 4837281 INFO  (qtp2053050546-159713) [n:127.0.0.1:42087_rqj%2Fxd c:control_collection s:shard1 r:core_node2 x:control_collection_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/control_collection/leaders/shard1/leader after winning as /collections/control_collection/leader_elect/shard1/election/72302333454909444-core_node2-n_0000000000
   [junit4]   2> 4837283 INFO  (qtp2053050546-159713) [n:127.0.0.1:42087_rqj%2Fxd c:control_collection s:shard1 r:core_node2 x:control_collection_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:42087/rqj/xd/control_collection_shard1_replica_n1/ shard1
   [junit4]   2> 4837385 INFO  (zkCallback-31923-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 4837385 INFO  (zkCallback-31923-thread-2) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 4837386 INFO  (qtp2053050546-159713) [n:127.0.0.1:42087_rqj%2Fxd c:control_collection s:shard1 r:core_node2 x:control_collection_shard1_replica_n1 ] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 4837388 INFO  (qtp2053050546-159713) [n:127.0.0.1:42087_rqj%2Fxd     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node2&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=352
   [junit4]   2> 4837390 INFO  (qtp2053050546-159710) [n:127.0.0.1:42087_rqj%2Fxd     ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
   [junit4]   2> 4837489 INFO  (zkCallback-31923-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 4837489 INFO  (zkCallback-31923-thread-3) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 4837489 INFO  (zkCallback-31923-thread-2) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 4837490 INFO  (qtp2053050546-159710) [n:127.0.0.1:42087_rqj%2Fxd     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:42087_rqj%252Fxd&wt=javabin&version=2} status=0 QTime=768
   [junit4]   2> 4837490 INFO  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [     ] o.a.s.c.AbstractFullDistribZkTestBase Waiting to see 1 active replicas in collection: control_collection
   [junit4]   2> 4837492 WARN  (NIOWorkerThread-8) [     ] o.a.z.s.NIOServerCnxn Unexpected exception
   [junit4]   2>           => EndOfStreamException: Unable to read additional data from client, it probably closed the socket: address = /127.0.0.1:57366, session = 0x100de96d6030005
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.handleFailedRead(NIOServerCnxn.java:163)
   [junit4]   2> org.apache.zookeeper.server.ServerCnxn$EndOfStreamException: Unable to read additional data from client, it probably closed the socket: address = /127.0.0.1:57366, session = 0x100de96d6030005
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.handleFailedRead(NIOServerCnxn.java:163) ~[zookeeper-3.6.1.jar:3.6.1]
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:326) [zookeeper-3.6.1.jar:3.6.1]
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory$IOWorkRequest.doWork(NIOServerCnxnFactory.java:522) [zookeeper-3.6.1.jar:3.6.1]
   [junit4]   2> 	at org.apache.zookeeper.server.WorkerService$ScheduledWorkRequest.run(WorkerService.java:154) [zookeeper-3.6.1.jar:3.6.1]
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:834) [?:?]
   [junit4]   2> 4837595 INFO  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 4837596 INFO  (zkConnectionManagerCallback-31950-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 4837596 INFO  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 4837597 INFO  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 4837598 INFO  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [     ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:41665/solr ready
   [junit4]   2> 4837598 INFO  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [     ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 4837599 INFO  (qtp2053050546-159713) [n:127.0.0.1:42087_rqj%2Fxd     ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 4837605 INFO  (OverseerThreadFactory-31929-thread-2-processing-n:127.0.0.1:42087_rqj%2Fxd) [n:127.0.0.1:42087_rqj%2Fxd     ] o.a.s.c.a.c.CreateCollectionCmd Create collection collection1
   [junit4]   2> 4837605 INFO  (OverseerCollectionConfigSetProcessor-72302333454909444-127.0.0.1:42087_rqj%2Fxd-n_0000000000) [n:127.0.0.1:42087_rqj%2Fxd     ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
   [junit4]   2> 4837810 WARN  (OverseerThreadFactory-31929-thread-2-processing-n:127.0.0.1:42087_rqj%2Fxd) [n:127.0.0.1:42087_rqj%2Fxd     ] o.a.s.c.a.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
   [junit4]   2> 4837811 INFO  (qtp2053050546-159713) [n:127.0.0.1:42087_rqj%2Fxd     ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
   [junit4]   2> 4837812 INFO  (qtp2053050546-159713) [n:127.0.0.1:42087_rqj%2Fxd     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&wt=javabin&version=2} status=0 QTime=213
   [junit4]   2> 4837813 INFO  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [     ] o.a.s.c.SolrCloudTestCase active slice count: 2 expected: 2
   [junit4]   2> 4837813 INFO  (TEST-TestBulkSchemaConcurrent.test-seed#[1F01E002FCAD9B32]) [     ] o.a.s.c.SolrCloudTestCase active r

[...truncated too long message...]

FO  (coreCloseExecutor-32773-thread-1) [     ] o.a.s.c.SolrCore [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@7069220f
   [junit4]   2> 4957595 INFO  (coreCloseExecutor-32773-thread-1) [     ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1 tag=SolrCore@7069220f
   [junit4]   2> 4957595 INFO  (coreCloseExecutor-32773-thread-1) [     ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@1e369963: rootName = null, domain = solr.core.collection1, service url = null, agent id = null] for registry solr.core.collection1/com.codahale.metrics.MetricRegistry@68d0a3c4
   [junit4]   2> 4957607 INFO  (coreCloseExecutor-32773-thread-1) [     ] o.a.s.u.DirectUpdateHandler2 Committing on IndexWriter.close()  ... SKIPPED (unnecessary).
   [junit4]   2> 4957669 INFO  (SUITE-TestReRankQParserPlugin-seed#[1F01E002FCAD9B32]-worker) [     ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node tag=null
   [junit4]   2> 4957669 INFO  (SUITE-TestReRankQParserPlugin-seed#[1F01E002FCAD9B32]-worker) [     ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@5e8f08f3: rootName = null, domain = solr.node, service url = null, agent id = null] for registry solr.node/com.codahale.metrics.MetricRegistry@3a88664d
   [junit4]   2> 4957674 INFO  (SUITE-TestReRankQParserPlugin-seed#[1F01E002FCAD9B32]-worker) [     ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm tag=null
   [junit4]   2> 4957674 INFO  (SUITE-TestReRankQParserPlugin-seed#[1F01E002FCAD9B32]-worker) [     ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@4c5dfd28: rootName = null, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm/com.codahale.metrics.MetricRegistry@7f2e4611
   [junit4]   2> 4957676 INFO  (SUITE-TestReRankQParserPlugin-seed#[1F01E002FCAD9B32]-worker) [     ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty tag=null
   [junit4]   2> 4957676 INFO  (SUITE-TestReRankQParserPlugin-seed#[1F01E002FCAD9B32]-worker) [     ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@7404a2d7: rootName = null, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty/com.codahale.metrics.MetricRegistry@5146c0c4
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.search.TestReRankQParserPlugin_1F01E002FCAD9B32-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene86): {field_t=BlockTreeOrds(blocksize=128), term_s=PostingsFormat(name=LuceneVarGapDocFreqInterval), id=Lucene84, group_s=Lucene84}, docValues:{}, maxPointsInLeafNode=1427, maxMBSortInHeap=6.740644465885312, sim=Asserting(RandomSimilarity(queryNorm=false): {}), locale=en-SE, timezone=Etc/GMT+7
   [junit4]   2> NOTE: Linux 4.15.0-108-generic amd64/Oracle Corporation 11.0.6 (64-bit)/cpus=4,threads=1,free=154052872,total=536870912
   [junit4]   2> NOTE: All tests run in this JVM: [TestUninvertingReader, TestSolrConfigHandler, TestHdfsBackupRestoreCore, TestSolrTestCaseJ4, TestJmxIntegration, HDFSCollectionsAPITest, TestDynamicFieldNamesIndexCorrectly, TestStreamBody, TestSortByMinMaxFunction, TestConfigOverlay, TestRandomFaceting, DistributedSpellCheckComponentTest, ForceLeaderWithTlogReplicasTest, TestAuthorizationFramework, TestRuleBasedAuthorizationWithKerberos, TestRequestRateLimiter, PrimUtilsTest, SolrCloudReportersTest, TestExportWriter, HdfsCollectionsAPIDistributedZkTest, TestPrepRecovery, TestSolrQueryParser, TestCloudManagedSchema, TestCorePropertiesReload, TestConfigReload, ReplaceNodeTest, JWTAuthPluginTest, TestHighlightDedupGrouping, HdfsLockFactoryTest, TestDistribPackageStore, TestPerFieldSimilarityWithDefaultOverride, TestDeprecatedFilters, XsltUpdateRequestHandlerTest, TestHdfsUpdateLog, TestClassicSimilarityFactory, DistribDocExpirationUpdateProcessorTest, TestSolr4Spatial, TestSQLHandler, TaggingAttributeTest, DistributedQueryComponentCustomSortTest, TestSolrCLIRunExample, TestSchemalessBufferedUpdates, TestCloudPseudoReturnFields, TestLegacyFieldReuse, SortSpecParsingTest, SuggesterTSTTest, SolrLogPostToolTest, BlobRepositoryCloudTest, NestedShardedAtomicUpdateTest, BJQFilterAccessibleTest, TestRawResponseWriter, TestLockTree, IgnoreLargeDocumentProcessorFactoryTest, TestSchemaVersionResource, TestMacroExpander, TestQueryingOnDownCollection, TestSSLRandomization, ExternalFileFieldSortTest, TestDefaultStatsCache, TestCoreAdminApis, LeaderFailureAfterFreshStartTest, TestTestInjection, TestLegacyFieldCache, TestMultiValuedNumericRangeQuery, AtomicUpdatesTest, TestSchemaNameResource, TestFieldSortValues, TestComplexPhraseQParserPlugin, MetricUtilsTest, HLLUtilTest, SolrJmxReporterCloudTest, PeerSyncWithLeaderTest, ZookeeperStatusHandlerTest, TestInPlaceUpdatesRequiredField, ResponseLogComponentTest, ConcurrentCreateRoutedAliasTest, TestClusterStateMutator, TestFieldResource, DirectoryFactoryTest, TestCursorMarkWithoutUniqueKey, BasicDistributedZk2Test, SmileWriterTest, TestMultiWordSynonyms, TestJsonFacetsWithNestedObjects, DistributedDebugComponentTest, CachingDirectoryFactoryTest, TestHttpShardHandlerFactory, TestNestedUpdateProcessor, LeaderElectionIntegrationTest, CollectionPropsTest, TestTermsQParserPlugin, TestDynamicFieldCollectionResource, UtilsToolTest, ClassificationUpdateProcessorIntegrationTest, HdfsRecoverLeaseTest, ZkControllerTest, FacetPivotSmallTest, TolerantUpdateProcessorTest, TestSolrCoreParser, TestCaffeineCache, UnloadDistributedZkTest, TestFieldCacheSortRandom, TestManagedSchemaAPI, SaslZkACLProviderTest, SimpleFacetsTest, TestSolrConfigHandlerConcurrent, TestStressUserVersions, BigEndianAscendingWordDeserializerTest, DateFieldTest, HdfsChaosMonkeyNothingIsSafeTest, SolrIndexSplitterTest, HighlighterConfigTest, ManagedSchemaRoundRobinCloudTest, TestSizeLimitedDistributedMap, ZkStateWriterTest, SearchHandlerTest, MetricsHistoryIntegrationTest, CollectionsAPIDistributedZkTest, TestTlogReplica, TestIndexSearcher, PreAnalyzedFieldTest, BadCopyFieldTest, TestUnifiedSolrHighlighterWithoutStoredId, SyncSliceTest, TestMaxScoreQueryParser, HttpPartitionTest, SuggestComponentTest, TestReplicationHandlerDiskOverFlow, TestApiFramework, FileUtilsTest, SolrCmdDistributorTest, TestStressLucene, TestEmbeddedSolrServerAdminHandler, TestLeaderElectionWithEmptyReplica, DocExpirationUpdateProcessorFactoryTest, TestDocSet, DistributedVersionInfoTest, TestRandomCollapseQParserPlugin, AuditLoggerPluginTest, TestXmlQParser, AnalysisAfterCoreReloadTest, BasicFunctionalityTest, ConvertedLegacyTest, CursorPagingTest, DisMaxRequestHandlerTest, DistributedIntervalFacetingTest, EchoParamsTest, MinimalSchemaTest, OutputWriterTest, SampleTest, SolrInfoBeanTest, SolrTestCaseJ4Test, TestCrossCoreJoin, TestDistributedMissingSort, TestDistributedSearch, TestDocumentBuilder, TestGroupingSearch, TestTrie, PathHierarchyTokenizerFactoryTest, ProtectedTermFilterFactoryTest, TestCharFilters, TestReversedWildcardFilterFactory, TestWordDelimiterFilterFactory, TokenizerChainTest, TestJettySolrRunner, ConnectionReuseTest, AddReplicaTest, AliasIntegrationTest, ChaosMonkeyNothingIsSafeWithPullReplicasTest, ChaosMonkeySafeLeaderTest, CleanupOldIndexTest, CloudExitableDirectoryReaderTest, ClusterStateMockUtilTest, ClusterStateTest, CollectionsAPISolrJTest, DeleteShardTest, DeleteStatusTest, LeaderElectionContextKeyTest, LeaderVoteWaitTimeoutTest, MoveReplicaTest, MultiSolrCloudTestCaseTest, OverriddenZkACLAndCredentialsProvidersTest, OverseerRolesTest, OverseerTest, ReindexCollectionTest, SolrXmlInZkTest, TestCloudConsistency, TestClusterProperties, TestConfigSetsAPI, TestDistributedMap, TestDownShardTolerantSearch, TestHashPartitioner, TestRebalanceLeaders, TestStressCloudBlindAtomicUpdates, TriLevelCompositeIdRoutingTest, ZkShardTermsTest, ZkSolrClientTest, AsyncCallRequestStatusResponseTest, CollectionReloadTest, CollectionTooManyReplicasTest, CollectionsAPIAsyncDistributedZkTest, TestCollectionsAPIViaSolrCloudCluster, TestHdfsCloudBackupRestore, BaseCdcrDistributedZkTest, CdcrBootstrapTest, CdcrRequestHandlerTest, CdcrVersionReplicationTest, HdfsBasicDistributedZkTest, TestLegacyTerms, TestNumericRangeQuery64, SolrMetricReporterTest, SolrMetricsIntegrationTest, SolrSlf4jReporterTest, TestRemoteStreaming, TestCustomDocTransformer, TestGeoJSONResponseWriter, TestGraphMLResponseWriter, TestJavabinTupleStreamParser, TestPHPSerializedResponseWriter, TestSolrQueryResponse, TestChildDocTransformer, TestExplainDocTransformer, TestSubQueryTransformer, TestSubQueryTransformerCrossCore, TestSubQueryTransformerDistrib, TestManagedResource, TestRestManager, TestBulkSchemaAPI, TestFieldTypeResource, TestSchemaSimilarityResource, TestUniqueKeyFieldResource, TestManagedStopFilterFactory, TestManagedSynonymFilterFactory, TestManagedSynonymGraphFilterFactory, ChangedSchemaMergeTest, CopyFieldTest, CurrencyFieldTypeTest, DocValuesMissingTest, DocValuesMultiTest, DocValuesTest, EnumFieldTest, OpenExchangeRatesOrgProviderTest, PolyFieldTest, RankFieldTest, RequiredFieldsTest, ResolveAnalyzerByNameTest, SchemaVersionSpecificBehaviorTest, SpatialRPTFieldTypeTest, SynonymTokenizerTest, TestBinaryField, TestBulkSchemaConcurrent, TestUseDocValuesAsStored2, UUIDFieldTest, AnalyticsMergeStrategyTest, AnalyticsQueryTest, CursorMarkTest, FuzzySearchTest, MaxScoreCollectorTest, MergeStrategyTest, QueryParsingTest, RankQParserPluginTest, RankQueryTest, ReturnFieldsTest, SignificantTermsQParserPluginTest, TestAddFieldRealTimeGet, TestCustomSort, TestFiltering, TestPayloadCheckQParserPlugin, TestPseudoReturnFields, TestQueryUtils, TestRTGBase, TestReRankQParserPlugin]
   [junit4] Completed [736/865 (2!)] on J1 in 0.93s, 6 tests, 1 error <<< FAILURES!

[...truncated 52304 lines...]

[JENKINS] Lucene » Lucene-Solr-NightlyTests-master - Build # 5 - Still Unstable!

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://ci-builds.apache.org/job/Lucene/job/Lucene-Solr-NightlyTests-master/5/

3 tests failed.
FAILED:  org.apache.solr.cloud.api.collections.TestHdfsCloudBackupRestore.test

Error Message:
Node 127.0.0.1:38391_solr has 9 replicas. Expected num replicas : 5. state:  DocCollection(hdfsbackuprestore_testok_restored/32)={   "pullReplicas":1,   "replicationFactor":1,   "shards":{     "shard2":{       "range":"0-7fffffff",       "state":"active",       "replicas":{         "core_node2":{           "core":"hdfsbackuprestore_testok_restored_shard2_replica_n1",           "leader":"true",           "core_node_name":"core_node2",           "base_url":"http://127.0.0.1:38391/solr",           "node_name":"127.0.0.1:38391_solr",           "state":"active",           "collection":"hdfsbackuprestore_testok_restored",           "shard":"shard2",           "type":"NRT",           "force_set_state":"false"},         "core_node8":{           "core":"hdfsbackuprestore_testok_restored_shard2_replica_t7",           "core_node_name":"core_node8",           "base_url":"http://127.0.0.1:38391/solr",           "node_name":"127.0.0.1:38391_solr",           "state":"active",           "collection":"hdfsbackuprestore_testok_restored",           "shard":"shard2",           "type":"TLOG",           "force_set_state":"false"},         "core_node10":{           "core":"hdfsbackuprestore_testok_restored_shard2_replica_p9",           "core_node_name":"core_node10",           "base_url":"http://127.0.0.1:38391/solr",           "node_name":"127.0.0.1:38391_solr",           "state":"active",           "collection":"hdfsbackuprestore_testok_restored",           "shard":"shard2",           "type":"PULL",           "force_set_state":"false"}},       "stateTimestamp":"1597138944990830757"},     "shard1_1":{       "range":"c0000000-ffffffff",       "state":"active",       "replicas":{         "core_node4":{           "core":"hdfsbackuprestore_testok_restored_shard1_1_replica_n3",           "leader":"true",           "core_node_name":"core_node4",           "base_url":"http://127.0.0.1:38391/solr",           "node_name":"127.0.0.1:38391_solr",           "state":"active",           "collection":"hdfsbackuprestore_testok_restored",           "shard":"shard1_1",           "type":"NRT",           "force_set_state":"false"},         "core_node12":{           "core":"hdfsbackuprestore_testok_restored_shard1_1_replica_t11",           "core_node_name":"core_node12",           "base_url":"http://127.0.0.1:38391/solr",           "node_name":"127.0.0.1:38391_solr",           "state":"active",           "collection":"hdfsbackuprestore_testok_restored",           "shard":"shard1_1",           "type":"TLOG",           "force_set_state":"false"},         "core_node14":{           "core":"hdfsbackuprestore_testok_restored_shard1_1_replica_p13",           "core_node_name":"core_node14",           "base_url":"http://127.0.0.1:38391/solr",           "node_name":"127.0.0.1:38391_solr",           "state":"active",           "collection":"hdfsbackuprestore_testok_restored",           "shard":"shard1_1",           "type":"PULL",           "force_set_state":"false"}},       "stateTimestamp":"1597138944990860837"},     "shard1_0":{       "range":"80000000-bfffffff",       "state":"active",       "replicas":{         "core_node6":{           "core":"hdfsbackuprestore_testok_restored_shard1_0_replica_n5",           "leader":"true",           "core_node_name":"core_node6",           "base_url":"http://127.0.0.1:38391/solr",           "node_name":"127.0.0.1:38391_solr",           "state":"active",           "collection":"hdfsbackuprestore_testok_restored",           "shard":"shard1_0",           "type":"NRT",           "force_set_state":"false"},         "core_node16":{           "core":"hdfsbackuprestore_testok_restored_shard1_0_replica_t15",           "core_node_name":"core_node16",           "base_url":"http://127.0.0.1:38391/solr",           "node_name":"127.0.0.1:38391_solr",           "state":"active",           "collection":"hdfsbackuprestore_testok_restored",           "shard":"shard1_0",           "type":"TLOG",           "force_set_state":"false"},         "core_node18":{           "core":"hdfsbackuprestore_testok_restored_shard1_0_replica_p17",           "core_node_name":"core_node18",           "base_url":"http://127.0.0.1:38391/solr",           "node_name":"127.0.0.1:38391_solr",           "state":"active",           "collection":"hdfsbackuprestore_testok_restored",           "shard":"shard1_0",           "type":"PULL",           "force_set_state":"false"}},       "stateTimestamp":"1597138944990913053"}},   "router":{     "name":"compositeId",     "field":"shard_s"},   "nrtReplicas":1,   "tlogReplicas":1}

Stack Trace:
java.lang.AssertionError: Node 127.0.0.1:38391_solr has 9 replicas. Expected num replicas : 5. state: 
DocCollection(hdfsbackuprestore_testok_restored/32)={
  "pullReplicas":1,
  "replicationFactor":1,
  "shards":{
    "shard2":{
      "range":"0-7fffffff",
      "state":"active",
      "replicas":{
        "core_node2":{
          "core":"hdfsbackuprestore_testok_restored_shard2_replica_n1",
          "leader":"true",
          "core_node_name":"core_node2",
          "base_url":"http://127.0.0.1:38391/solr",
          "node_name":"127.0.0.1:38391_solr",
          "state":"active",
          "collection":"hdfsbackuprestore_testok_restored",
          "shard":"shard2",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node8":{
          "core":"hdfsbackuprestore_testok_restored_shard2_replica_t7",
          "core_node_name":"core_node8",
          "base_url":"http://127.0.0.1:38391/solr",
          "node_name":"127.0.0.1:38391_solr",
          "state":"active",
          "collection":"hdfsbackuprestore_testok_restored",
          "shard":"shard2",
          "type":"TLOG",
          "force_set_state":"false"},
        "core_node10":{
          "core":"hdfsbackuprestore_testok_restored_shard2_replica_p9",
          "core_node_name":"core_node10",
          "base_url":"http://127.0.0.1:38391/solr",
          "node_name":"127.0.0.1:38391_solr",
          "state":"active",
          "collection":"hdfsbackuprestore_testok_restored",
          "shard":"shard2",
          "type":"PULL",
          "force_set_state":"false"}},
      "stateTimestamp":"1597138944990830757"},
    "shard1_1":{
      "range":"c0000000-ffffffff",
      "state":"active",
      "replicas":{
        "core_node4":{
          "core":"hdfsbackuprestore_testok_restored_shard1_1_replica_n3",
          "leader":"true",
          "core_node_name":"core_node4",
          "base_url":"http://127.0.0.1:38391/solr",
          "node_name":"127.0.0.1:38391_solr",
          "state":"active",
          "collection":"hdfsbackuprestore_testok_restored",
          "shard":"shard1_1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node12":{
          "core":"hdfsbackuprestore_testok_restored_shard1_1_replica_t11",
          "core_node_name":"core_node12",
          "base_url":"http://127.0.0.1:38391/solr",
          "node_name":"127.0.0.1:38391_solr",
          "state":"active",
          "collection":"hdfsbackuprestore_testok_restored",
          "shard":"shard1_1",
          "type":"TLOG",
          "force_set_state":"false"},
        "core_node14":{
          "core":"hdfsbackuprestore_testok_restored_shard1_1_replica_p13",
          "core_node_name":"core_node14",
          "base_url":"http://127.0.0.1:38391/solr",
          "node_name":"127.0.0.1:38391_solr",
          "state":"active",
          "collection":"hdfsbackuprestore_testok_restored",
          "shard":"shard1_1",
          "type":"PULL",
          "force_set_state":"false"}},
      "stateTimestamp":"1597138944990860837"},
    "shard1_0":{
      "range":"80000000-bfffffff",
      "state":"active",
      "replicas":{
        "core_node6":{
          "core":"hdfsbackuprestore_testok_restored_shard1_0_replica_n5",
          "leader":"true",
          "core_node_name":"core_node6",
          "base_url":"http://127.0.0.1:38391/solr",
          "node_name":"127.0.0.1:38391_solr",
          "state":"active",
          "collection":"hdfsbackuprestore_testok_restored",
          "shard":"shard1_0",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node16":{
          "core":"hdfsbackuprestore_testok_restored_shard1_0_replica_t15",
          "core_node_name":"core_node16",
          "base_url":"http://127.0.0.1:38391/solr",
          "node_name":"127.0.0.1:38391_solr",
          "state":"active",
          "collection":"hdfsbackuprestore_testok_restored",
          "shard":"shard1_0",
          "type":"TLOG",
          "force_set_state":"false"},
        "core_node18":{
          "core":"hdfsbackuprestore_testok_restored_shard1_0_replica_p17",
          "core_node_name":"core_node18",
          "base_url":"http://127.0.0.1:38391/solr",
          "node_name":"127.0.0.1:38391_solr",
          "state":"active",
          "collection":"hdfsbackuprestore_testok_restored",
          "shard":"shard1_0",
          "type":"PULL",
          "force_set_state":"false"}},
      "stateTimestamp":"1597138944990913053"}},
  "router":{
    "name":"compositeId",
    "field":"shard_s"},
  "nrtReplicas":1,
  "tlogReplicas":1}
	at __randomizedtesting.SeedInfo.seed([D075C04A1EF5F5CE:5821FF90B0099836]:0)
	at org.junit.Assert.fail(Assert.java:88)
	at org.junit.Assert.assertTrue(Assert.java:41)
	at org.apache.solr.cloud.api.collections.AbstractCloudBackupRestoreTestCase.lambda$testBackupAndRestore$1(AbstractCloudBackupRestoreTestCase.java:374)
	at java.base/java.util.HashMap.forEach(HashMap.java:1336)
	at org.apache.solr.cloud.api.collections.AbstractCloudBackupRestoreTestCase.testBackupAndRestore(AbstractCloudBackupRestoreTestCase.java:373)
	at org.apache.solr.cloud.api.collections.AbstractCloudBackupRestoreTestCase.test(AbstractCloudBackupRestoreTestCase.java:155)
	at org.apache.solr.cloud.api.collections.TestHdfsCloudBackupRestore.test(TestHdfsCloudBackupRestore.java:225)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:834)


FAILED:  org.apache.solr.cloud.api.collections.TestLocalFSCloudBackupRestore.test

Error Message:
Node 127.0.0.1:37311_solr has 9 replicas. Expected num replicas : 5. state:  DocCollection(backuprestore_testok_restored/30)={   "pullReplicas":1,   "replicationFactor":1,   "shards":{     "shard2":{       "range":"0-7fffffff",       "state":"active",       "replicas":{         "core_node2":{           "core":"backuprestore_testok_restored_shard2_replica_n1",           "leader":"true",           "core_node_name":"core_node2",           "base_url":"https://127.0.0.1:37311/solr",           "node_name":"127.0.0.1:37311_solr",           "state":"active",           "collection":"backuprestore_testok_restored",           "shard":"shard2",           "type":"NRT",           "force_set_state":"false"},         "core_node8":{           "core":"backuprestore_testok_restored_shard2_replica_t7",           "core_node_name":"core_node8",           "base_url":"https://127.0.0.1:37311/solr",           "node_name":"127.0.0.1:37311_solr",           "state":"active",           "collection":"backuprestore_testok_restored",           "shard":"shard2",           "type":"TLOG",           "force_set_state":"false"},         "core_node10":{           "core":"backuprestore_testok_restored_shard2_replica_p9",           "core_node_name":"core_node10",           "base_url":"https://127.0.0.1:37311/solr",           "node_name":"127.0.0.1:37311_solr",           "state":"active",           "collection":"backuprestore_testok_restored",           "shard":"shard2",           "type":"PULL",           "force_set_state":"false"}},       "stateTimestamp":"1597138973336535008"},     "shard1_1":{       "range":"c0000000-ffffffff",       "state":"active",       "replicas":{         "core_node4":{           "core":"backuprestore_testok_restored_shard1_1_replica_n3",           "leader":"true",           "core_node_name":"core_node4",           "base_url":"https://127.0.0.1:37311/solr",           "node_name":"127.0.0.1:37311_solr",           "state":"active",           "collection":"backuprestore_testok_restored",           "shard":"shard1_1",           "type":"NRT",           "force_set_state":"false"},         "core_node12":{           "core":"backuprestore_testok_restored_shard1_1_replica_t11",           "core_node_name":"core_node12",           "base_url":"https://127.0.0.1:37311/solr",           "node_name":"127.0.0.1:37311_solr",           "state":"active",           "collection":"backuprestore_testok_restored",           "shard":"shard1_1",           "type":"TLOG",           "force_set_state":"false"},         "core_node14":{           "core":"backuprestore_testok_restored_shard1_1_replica_p13",           "core_node_name":"core_node14",           "base_url":"https://127.0.0.1:37311/solr",           "node_name":"127.0.0.1:37311_solr",           "state":"active",           "collection":"backuprestore_testok_restored",           "shard":"shard1_1",           "type":"PULL",           "force_set_state":"false"}},       "stateTimestamp":"1597138973336568907"},     "shard1_0":{       "range":"80000000-bfffffff",       "state":"active",       "replicas":{         "core_node6":{           "core":"backuprestore_testok_restored_shard1_0_replica_n5",           "leader":"true",           "core_node_name":"core_node6",           "base_url":"https://127.0.0.1:37311/solr",           "node_name":"127.0.0.1:37311_solr",           "state":"active",           "collection":"backuprestore_testok_restored",           "shard":"shard1_0",           "type":"NRT",           "force_set_state":"false"},         "core_node16":{           "core":"backuprestore_testok_restored_shard1_0_replica_t15",           "core_node_name":"core_node16",           "base_url":"https://127.0.0.1:37311/solr",           "node_name":"127.0.0.1:37311_solr",           "state":"active",           "collection":"backuprestore_testok_restored",           "shard":"shard1_0",           "type":"TLOG",           "force_set_state":"false"},         "core_node18":{           "core":"backuprestore_testok_restored_shard1_0_replica_p17",           "core_node_name":"core_node18",           "base_url":"https://127.0.0.1:37311/solr",           "node_name":"127.0.0.1:37311_solr",           "state":"active",           "collection":"backuprestore_testok_restored",           "shard":"shard1_0",           "type":"PULL",           "force_set_state":"false"}},       "stateTimestamp":"1597138973336592599"}},   "router":{     "name":"compositeId",     "field":"shard_s"},   "nrtReplicas":1,   "tlogReplicas":1}

Stack Trace:
java.lang.AssertionError: Node 127.0.0.1:37311_solr has 9 replicas. Expected num replicas : 5. state: 
DocCollection(backuprestore_testok_restored/30)={
  "pullReplicas":1,
  "replicationFactor":1,
  "shards":{
    "shard2":{
      "range":"0-7fffffff",
      "state":"active",
      "replicas":{
        "core_node2":{
          "core":"backuprestore_testok_restored_shard2_replica_n1",
          "leader":"true",
          "core_node_name":"core_node2",
          "base_url":"https://127.0.0.1:37311/solr",
          "node_name":"127.0.0.1:37311_solr",
          "state":"active",
          "collection":"backuprestore_testok_restored",
          "shard":"shard2",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node8":{
          "core":"backuprestore_testok_restored_shard2_replica_t7",
          "core_node_name":"core_node8",
          "base_url":"https://127.0.0.1:37311/solr",
          "node_name":"127.0.0.1:37311_solr",
          "state":"active",
          "collection":"backuprestore_testok_restored",
          "shard":"shard2",
          "type":"TLOG",
          "force_set_state":"false"},
        "core_node10":{
          "core":"backuprestore_testok_restored_shard2_replica_p9",
          "core_node_name":"core_node10",
          "base_url":"https://127.0.0.1:37311/solr",
          "node_name":"127.0.0.1:37311_solr",
          "state":"active",
          "collection":"backuprestore_testok_restored",
          "shard":"shard2",
          "type":"PULL",
          "force_set_state":"false"}},
      "stateTimestamp":"1597138973336535008"},
    "shard1_1":{
      "range":"c0000000-ffffffff",
      "state":"active",
      "replicas":{
        "core_node4":{
          "core":"backuprestore_testok_restored_shard1_1_replica_n3",
          "leader":"true",
          "core_node_name":"core_node4",
          "base_url":"https://127.0.0.1:37311/solr",
          "node_name":"127.0.0.1:37311_solr",
          "state":"active",
          "collection":"backuprestore_testok_restored",
          "shard":"shard1_1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node12":{
          "core":"backuprestore_testok_restored_shard1_1_replica_t11",
          "core_node_name":"core_node12",
          "base_url":"https://127.0.0.1:37311/solr",
          "node_name":"127.0.0.1:37311_solr",
          "state":"active",
          "collection":"backuprestore_testok_restored",
          "shard":"shard1_1",
          "type":"TLOG",
          "force_set_state":"false"},
        "core_node14":{
          "core":"backuprestore_testok_restored_shard1_1_replica_p13",
          "core_node_name":"core_node14",
          "base_url":"https://127.0.0.1:37311/solr",
          "node_name":"127.0.0.1:37311_solr",
          "state":"active",
          "collection":"backuprestore_testok_restored",
          "shard":"shard1_1",
          "type":"PULL",
          "force_set_state":"false"}},
      "stateTimestamp":"1597138973336568907"},
    "shard1_0":{
      "range":"80000000-bfffffff",
      "state":"active",
      "replicas":{
        "core_node6":{
          "core":"backuprestore_testok_restored_shard1_0_replica_n5",
          "leader":"true",
          "core_node_name":"core_node6",
          "base_url":"https://127.0.0.1:37311/solr",
          "node_name":"127.0.0.1:37311_solr",
          "state":"active",
          "collection":"backuprestore_testok_restored",
          "shard":"shard1_0",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node16":{
          "core":"backuprestore_testok_restored_shard1_0_replica_t15",
          "core_node_name":"core_node16",
          "base_url":"https://127.0.0.1:37311/solr",
          "node_name":"127.0.0.1:37311_solr",
          "state":"active",
          "collection":"backuprestore_testok_restored",
          "shard":"shard1_0",
          "type":"TLOG",
          "force_set_state":"false"},
        "core_node18":{
          "core":"backuprestore_testok_restored_shard1_0_replica_p17",
          "core_node_name":"core_node18",
          "base_url":"https://127.0.0.1:37311/solr",
          "node_name":"127.0.0.1:37311_solr",
          "state":"active",
          "collection":"backuprestore_testok_restored",
          "shard":"shard1_0",
          "type":"PULL",
          "force_set_state":"false"}},
      "stateTimestamp":"1597138973336592599"}},
  "router":{
    "name":"compositeId",
    "field":"shard_s"},
  "nrtReplicas":1,
  "tlogReplicas":1}
	at __randomizedtesting.SeedInfo.seed([D075C04A1EF5F5CE:5821FF90B0099836]:0)
	at org.junit.Assert.fail(Assert.java:88)
	at org.junit.Assert.assertTrue(Assert.java:41)
	at org.apache.solr.cloud.api.collections.AbstractCloudBackupRestoreTestCase.lambda$testBackupAndRestore$1(AbstractCloudBackupRestoreTestCase.java:374)
	at java.base/java.util.HashMap.forEach(HashMap.java:1336)
	at org.apache.solr.cloud.api.collections.AbstractCloudBackupRestoreTestCase.testBackupAndRestore(AbstractCloudBackupRestoreTestCase.java:373)
	at org.apache.solr.cloud.api.collections.AbstractCloudBackupRestoreTestCase.test(AbstractCloudBackupRestoreTestCase.java:155)
	at org.apache.solr.cloud.api.collections.TestLocalFSCloudBackupRestore.test(TestLocalFSCloudBackupRestore.java:93)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:834)


FAILED:  org.apache.solr.servlet.TestRequestRateLimiter.testConcurrentQueries

Error Message:
Incoming accepted new request count did not match. Expected 5 incoming 25

Stack Trace:
java.lang.AssertionError: Incoming accepted new request count did not match. Expected 5 incoming 25
	at __randomizedtesting.SeedInfo.seed([D075C04A1EF5F5CE:92EE236F9E9280EF]:0)
	at org.junit.Assert.fail(Assert.java:88)
	at org.junit.Assert.assertTrue(Assert.java:41)
	at org.apache.solr.servlet.TestRequestRateLimiter.testConcurrentQueries(TestRequestRateLimiter.java:74)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:834)




Build Log:
[...truncated 12424 lines...]
   [junit4] Suite: org.apache.solr.servlet.TestRequestRateLimiter
   [junit4]   2> 201946 INFO  (SUITE-TestRequestRateLimiter-seed#[D075C04A1EF5F5CE]-worker) [     ] o.a.s.SolrTestCase Setting 'solr.default.confdir' system property to test-framework derived value of '/home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/server/solr/configsets/_default/conf'
   [junit4]   2> 201946 INFO  (SUITE-TestRequestRateLimiter-seed#[D075C04A1EF5F5CE]-worker) [     ] o.a.s.SolrTestCaseJ4 Created dataDir: /home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.servlet.TestRequestRateLimiter_D075C04A1EF5F5CE-001/data-dir-16-001
   [junit4]   2> 201947 WARN  (SUITE-TestRequestRateLimiter-seed#[D075C04A1EF5F5CE]-worker) [     ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=10 numCloses=10
   [junit4]   2> 201947 INFO  (SUITE-TestRequestRateLimiter-seed#[D075C04A1EF5F5CE]-worker) [     ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 201948 INFO  (SUITE-TestRequestRateLimiter-seed#[D075C04A1EF5F5CE]-worker) [     ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason="", ssl=0.0/0.0, value=0.0/0.0, clientAuth=0.0/0.0)
   [junit4]   2> 201948 INFO  (SUITE-TestRequestRateLimiter-seed#[D075C04A1EF5F5CE]-worker) [     ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 201948 INFO  (SUITE-TestRequestRateLimiter-seed#[D075C04A1EF5F5CE]-worker) [     ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 1 servers in /home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.servlet.TestRequestRateLimiter_D075C04A1EF5F5CE-001/tempDir-001
   [junit4]   2> 201948 INFO  (SUITE-TestRequestRateLimiter-seed#[D075C04A1EF5F5CE]-worker) [     ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 201949 INFO  (ZkTestServer Run Thread) [     ] o.a.s.c.ZkTestServer client port: 0.0.0.0/0.0.0.0:0
   [junit4]   2> 201949 INFO  (ZkTestServer Run Thread) [     ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 201950 WARN  (ZkTestServer Run Thread) [     ] o.a.z.s.ServerCnxnFactory maxCnxns is not configured, using default value 0.
   [junit4]   2> 202049 INFO  (SUITE-TestRequestRateLimiter-seed#[D075C04A1EF5F5CE]-worker) [     ] o.a.s.c.ZkTestServer start zk server on port: 38991
   [junit4]   2> 202049 INFO  (SUITE-TestRequestRateLimiter-seed#[D075C04A1EF5F5CE]-worker) [     ] o.a.s.c.ZkTestServer waitForServerUp: 127.0.0.1:38991
   [junit4]   2> 202049 INFO  (SUITE-TestRequestRateLimiter-seed#[D075C04A1EF5F5CE]-worker) [     ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:38991
   [junit4]   2> 202049 INFO  (SUITE-TestRequestRateLimiter-seed#[D075C04A1EF5F5CE]-worker) [     ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 38991
   [junit4]   2> 202052 INFO  (SUITE-TestRequestRateLimiter-seed#[D075C04A1EF5F5CE]-worker) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 202054 INFO  (zkConnectionManagerCallback-1804-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 202054 INFO  (SUITE-TestRequestRateLimiter-seed#[D075C04A1EF5F5CE]-worker) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 202060 INFO  (SUITE-TestRequestRateLimiter-seed#[D075C04A1EF5F5CE]-worker) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 202063 INFO  (zkConnectionManagerCallback-1806-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 202063 INFO  (SUITE-TestRequestRateLimiter-seed#[D075C04A1EF5F5CE]-worker) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 202064 INFO  (SUITE-TestRequestRateLimiter-seed#[D075C04A1EF5F5CE]-worker) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 202072 INFO  (zkConnectionManagerCallback-1808-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 202072 INFO  (SUITE-TestRequestRateLimiter-seed#[D075C04A1EF5F5CE]-worker) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 202178 INFO  (jetty-launcher-1809-thread-1) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
   [junit4]   2> 202178 INFO  (jetty-launcher-1809-thread-1) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 2 ...
   [junit4]   2> 202178 INFO  (jetty-launcher-1809-thread-1) [     ] o.e.j.s.Server jetty-9.4.27.v20200227; built: 2020-02-27T18:37:21.340Z; git: a304fd9f351f337e7c0e2a7c28878dd536149c6c; jvm 11.0.4+10-LTS
   [junit4]   2> 202179 INFO  (jetty-launcher-1809-thread-1) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 202179 INFO  (jetty-launcher-1809-thread-1) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 202179 INFO  (jetty-launcher-1809-thread-1) [     ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 202180 INFO  (jetty-launcher-1809-thread-1) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@22647689{/solr,null,AVAILABLE}
   [junit4]   2> 202180 INFO  (jetty-launcher-1809-thread-1) [     ] o.e.j.s.AbstractConnector Started ServerConnector@18b1eb96{HTTP/1.1, (http/1.1, h2c)}{127.0.0.1:36943}
   [junit4]   2> 202180 INFO  (jetty-launcher-1809-thread-1) [     ] o.e.j.s.Server Started @202218ms
   [junit4]   2> 202180 INFO  (jetty-launcher-1809-thread-1) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=36943}
   [junit4]   2> 202181 ERROR (jetty-launcher-1809-thread-1) [     ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 202181 INFO  (jetty-launcher-1809-thread-1) [     ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 202181 INFO  (jetty-launcher-1809-thread-1) [     ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 9.0.0
   [junit4]   2> 202181 INFO  (jetty-launcher-1809-thread-1) [     ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 202181 INFO  (jetty-launcher-1809-thread-1) [     ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: /home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr
   [junit4]   2> 202181 INFO  (jetty-launcher-1809-thread-1) [     ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2020-08-11T08:00:59.777225Z
   [junit4]   2> 202183 INFO  (jetty-launcher-1809-thread-1) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 202188 INFO  (zkConnectionManagerCallback-1811-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 202188 INFO  (jetty-launcher-1809-thread-1) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 202190 INFO  (jetty-launcher-1809-thread-1) [     ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 202297 INFO  (jetty-launcher-1809-thread-1) [     ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
   [junit4]   2> 202298 WARN  (jetty-launcher-1809-thread-1) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@7de57f51[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 202298 WARN  (jetty-launcher-1809-thread-1) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@7de57f51[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 202301 WARN  (jetty-launcher-1809-thread-1) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@50f30eb2[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 202301 WARN  (jetty-launcher-1809-thread-1) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@50f30eb2[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 202302 INFO  (jetty-launcher-1809-thread-1) [     ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:38991/solr
   [junit4]   2> 202303 INFO  (jetty-launcher-1809-thread-1) [n:127.0.0.1:36943_solr     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 202304 INFO  (zkConnectionManagerCallback-1821-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 202304 INFO  (jetty-launcher-1809-thread-1) [n:127.0.0.1:36943_solr     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 202423 INFO  (jetty-launcher-1809-thread-1) [n:127.0.0.1:36943_solr     ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:36943_solr
   [junit4]   2> 202424 INFO  (jetty-launcher-1809-thread-1) [n:127.0.0.1:36943_solr     ] o.a.s.c.Overseer Overseer (id=72079963249639428-127.0.0.1:36943_solr-n_0000000000) starting
   [junit4]   2> 202433 INFO  (OverseerStateUpdate-72079963249639428-127.0.0.1:36943_solr-n_0000000000) [n:127.0.0.1:36943_solr     ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:36943_solr
   [junit4]   2> 202433 INFO  (jetty-launcher-1809-thread-1) [n:127.0.0.1:36943_solr     ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:36943_solr
   [junit4]   2> 202434 INFO  (OverseerStateUpdate-72079963249639428-127.0.0.1:36943_solr-n_0000000000) [n:127.0.0.1:36943_solr     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 202438 INFO  (jetty-launcher-1809-thread-1) [n:127.0.0.1:36943_solr     ] o.a.s.p.PackageLoader /packages.json updated to version -1
   [junit4]   2> 202438 WARN  (jetty-launcher-1809-thread-1) [n:127.0.0.1:36943_solr     ] o.a.s.c.CoreContainer Not all security plugins configured!  authentication=disabled authorization=disabled.  Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external.  See https://s.apache.org/solrsecurity for more info
   [junit4]   2> 202454 INFO  (jetty-launcher-1809-thread-1) [n:127.0.0.1:36943_solr     ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 202480 INFO  (jetty-launcher-1809-thread-1) [n:127.0.0.1:36943_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_36943.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@148ebc3e
   [junit4]   2> 202490 INFO  (jetty-launcher-1809-thread-1) [n:127.0.0.1:36943_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_36943.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@148ebc3e
   [junit4]   2> 202490 INFO  (jetty-launcher-1809-thread-1) [n:127.0.0.1:36943_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_36943.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@148ebc3e
   [junit4]   2> 202492 INFO  (jetty-launcher-1809-thread-1) [n:127.0.0.1:36943_solr     ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.servlet.TestRequestRateLimiter_D075C04A1EF5F5CE-001/tempDir-001/node1
   [junit4]   2> 202497 INFO  (SUITE-TestRequestRateLimiter-seed#[D075C04A1EF5F5CE]-worker) [     ] o.a.s.c.MiniSolrCloudCluster waitForAllNodes: numServers=1
   [junit4]   2> 202498 INFO  (SUITE-TestRequestRateLimiter-seed#[D075C04A1EF5F5CE]-worker) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 202501 INFO  (zkConnectionManagerCallback-1836-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 202501 INFO  (SUITE-TestRequestRateLimiter-seed#[D075C04A1EF5F5CE]-worker) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 202502 INFO  (SUITE-TestRequestRateLimiter-seed#[D075C04A1EF5F5CE]-worker) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 202503 INFO  (SUITE-TestRequestRateLimiter-seed#[D075C04A1EF5F5CE]-worker) [     ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:38991/solr ready
   [junit4]   2> 202520 INFO  (TEST-TestRequestRateLimiter.testConcurrentQueries-seed#[D075C04A1EF5F5CE]) [     ] o.a.s.SolrTestCaseJ4 ###Starting testConcurrentQueries
   [junit4]   2> 202522 INFO  (qtp376837928-3249) [n:127.0.0.1:36943_solr     ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params pullReplicas=0&name=c1&nrtReplicas=1&action=CREATE&numShards=1&tlogReplicas=0&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 202526 INFO  (OverseerThreadFactory-1826-thread-1-processing-n:127.0.0.1:36943_solr) [n:127.0.0.1:36943_solr     ] o.a.s.c.a.c.CreateCollectionCmd Create collection c1
   [junit4]   2> 202527 INFO  (OverseerThreadFactory-1826-thread-1-processing-n:127.0.0.1:36943_solr) [n:127.0.0.1:36943_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/protwords.txt to /configs/c1.AUTOCREATED/protwords.txt
   [junit4]   2> 202529 INFO  (OverseerThreadFactory-1826-thread-1-processing-n:127.0.0.1:36943_solr) [n:127.0.0.1:36943_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/managed-schema to /configs/c1.AUTOCREATED/managed-schema
   [junit4]   2> 202531 INFO  (OverseerThreadFactory-1826-thread-1-processing-n:127.0.0.1:36943_solr) [n:127.0.0.1:36943_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/solrconfig.xml to /configs/c1.AUTOCREATED/solrconfig.xml
   [junit4]   2> 202534 INFO  (OverseerThreadFactory-1826-thread-1-processing-n:127.0.0.1:36943_solr) [n:127.0.0.1:36943_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/contractions_it.txt to /configs/c1.AUTOCREATED/lang/contractions_it.txt
   [junit4]   2> 202537 INFO  (OverseerThreadFactory-1826-thread-1-processing-n:127.0.0.1:36943_solr) [n:127.0.0.1:36943_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_no.txt to /configs/c1.AUTOCREATED/lang/stopwords_no.txt
   [junit4]   2> 202540 INFO  (OverseerThreadFactory-1826-thread-1-processing-n:127.0.0.1:36943_solr) [n:127.0.0.1:36943_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_hy.txt to /configs/c1.AUTOCREATED/lang/stopwords_hy.txt
   [junit4]   2> 202543 INFO  (OverseerThreadFactory-1826-thread-1-processing-n:127.0.0.1:36943_solr) [n:127.0.0.1:36943_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stemdict_nl.txt to /configs/c1.AUTOCREATED/lang/stemdict_nl.txt
   [junit4]   2> 202546 INFO  (OverseerThreadFactory-1826-thread-1-processing-n:127.0.0.1:36943_solr) [n:127.0.0.1:36943_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/contractions_ca.txt to /configs/c1.AUTOCREATED/lang/contractions_ca.txt
   [junit4]   2> 202549 INFO  (OverseerThreadFactory-1826-thread-1-processing-n:127.0.0.1:36943_solr) [n:127.0.0.1:36943_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_id.txt to /configs/c1.AUTOCREATED/lang/stopwords_id.txt
   [junit4]   2> 202552 INFO  (OverseerThreadFactory-1826-thread-1-processing-n:127.0.0.1:36943_solr) [n:127.0.0.1:36943_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_et.txt to /configs/c1.AUTOCREATED/lang/stopwords_et.txt
   [junit4]   2> 202554 INFO  (OverseerThreadFactory-1826-thread-1-processing-n:127.0.0.1:36943_solr) [n:127.0.0.1:36943_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_ga.txt to /configs/c1.AUTOCREATED/lang/stopwords_ga.txt
   [junit4]   2> 202557 INFO  (OverseerThreadFactory-1826-thread-1-processing-n:127.0.0.1:36943_solr) [n:127.0.0.1:36943_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_da.txt to /configs/c1.AUTOCREATED/lang/stopwords_da.txt
   [junit4]   2> 202560 INFO  (OverseerThreadFactory-1826-thread-1-processing-n:127.0.0.1:36943_solr) [n:127.0.0.1:36943_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_hi.txt to /configs/c1.AUTOCREATED/lang/stopwords_hi.txt
   [junit4]   2> 202563 INFO  (OverseerThreadFactory-1826-thread-1-processing-n:127.0.0.1:36943_solr) [n:127.0.0.1:36943_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_ja.txt to /configs/c1.AUTOCREATED/lang/stopwords_ja.txt
   [junit4]   2> 202566 INFO  (OverseerThreadFactory-1826-thread-1-processing-n:127.0.0.1:36943_solr) [n:127.0.0.1:36943_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_pt.txt to /configs/c1.AUTOCREATED/lang/stopwords_pt.txt
   [junit4]   2> 202569 INFO  (OverseerThreadFactory-1826-thread-1-processing-n:127.0.0.1:36943_solr) [n:127.0.0.1:36943_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_hu.txt to /configs/c1.AUTOCREATED/lang/stopwords_hu.txt
   [junit4]   2> 202572 INFO  (OverseerThreadFactory-1826-thread-1-processing-n:127.0.0.1:36943_solr) [n:127.0.0.1:36943_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_el.txt to /configs/c1.AUTOCREATED/lang/stopwords_el.txt
   [junit4]   2> 202574 INFO  (OverseerThreadFactory-1826-thread-1-processing-n:127.0.0.1:36943_solr) [n:127.0.0.1:36943_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_ru.txt to /configs/c1.AUTOCREATED/lang/stopwords_ru.txt
   [junit4]   2> 202576 INFO  (OverseerThreadFactory-1826-thread-1-processing-n:127.0.0.1:36943_solr) [n:127.0.0.1:36943_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_tr.txt to /configs/c1.AUTOCREATED/lang/stopwords_tr.txt
   [junit4]   2> 202579 INFO  (OverseerThreadFactory-1826-thread-1-processing-n:127.0.0.1:36943_solr) [n:127.0.0.1:36943_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_ar.txt to /configs/c1.AUTOCREATED/lang/stopwords_ar.txt
   [junit4]   2> 202581 INFO  (OverseerThreadFactory-1826-thread-1-processing-n:127.0.0.1:36943_solr) [n:127.0.0.1:36943_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/userdict_ja.txt to /configs/c1.AUTOCREATED/lang/userdict_ja.txt
   [junit4]   2> 202587 INFO  (OverseerThreadFactory-1826-thread-1-processing-n:127.0.0.1:36943_solr) [n:127.0.0.1:36943_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_it.txt to /configs/c1.AUTOCREATED/lang/stopwords_it.txt
   [junit4]   2> 202590 INFO  (OverseerThreadFactory-1826-thread-1-processing-n:127.0.0.1:36943_solr) [n:127.0.0.1:36943_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_eu.txt to /configs/c1.AUTOCREATED/lang/stopwords_eu.txt
   [junit4]   2> 202593 INFO  (OverseerThreadFactory-1826-thread-1-processing-n:127.0.0.1:36943_solr) [n:127.0.0.1:36943_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_cz.txt to /configs/c1.AUTOCREATED/lang/stopwords_cz.txt
   [junit4]   2> 202595 INFO  (OverseerThreadFactory-1826-thread-1-processing-n:127.0.0.1:36943_solr) [n:127.0.0.1:36943_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stoptags_ja.txt to /configs/c1.AUTOCREATED/lang/stoptags_ja.txt
   [junit4]   2> 202598 INFO  (OverseerThreadFactory-1826-thread-1-processing-n:127.0.0.1:36943_solr) [n:127.0.0.1:36943_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/contractions_fr.txt to /configs/c1.AUTOCREATED/lang/contractions_fr.txt
   [junit4]   2> 202601 INFO  (OverseerThreadFactory-1826-thread-1-processing-n:127.0.0.1:36943_solr) [n:127.0.0.1:36943_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_fa.txt to /configs/c1.AUTOCREATED/lang/stopwords_fa.txt
   [junit4]   2> 202603 INFO  (OverseerThreadFactory-1826-thread-1-processing-n:127.0.0.1:36943_solr) [n:127.0.0.1:36943_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_de.txt to /configs/c1.AUTOCREATED/lang/stopwords_de.txt
   [junit4]   2> 202606 INFO  (OverseerThreadFactory-1826-thread-1-processing-n:127.0.0.1:36943_solr) [n:127.0.0.1:36943_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/contractions_ga.txt to /configs/c1.AUTOCREATED/lang/contractions_ga.txt
   [junit4]   2> 202609 INFO  (OverseerThreadFactory-1826-thread-1-processing-n:127.0.0.1:36943_solr) [n:127.0.0.1:36943_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_ca.txt to /configs/c1.AUTOCREATED/lang/stopwords_ca.txt
   [junit4]   2> 202611 INFO  (OverseerThreadFactory-1826-thread-1-processing-n:127.0.0.1:36943_solr) [n:127.0.0.1:36943_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_nl.txt to /configs/c1.AUTOCREATED/lang/stopwords_nl.txt
   [junit4]   2> 202614 INFO  (OverseerThreadFactory-1826-thread-1-processing-n:127.0.0.1:36943_solr) [n:127.0.0.1:36943_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_bg.txt to /configs/c1.AUTOCREATED/lang/stopwords_bg.txt
   [junit4]   2> 202616 INFO  (OverseerThreadFactory-1826-thread-1-processing-n:127.0.0.1:36943_solr) [n:127.0.0.1:36943_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_es.txt to /configs/c1.AUTOCREATED/lang/stopwords_es.txt
   [junit4]   2> 202618 INFO  (OverseerThreadFactory-1826-thread-1-processing-n:127.0.0.1:36943_solr) [n:127.0.0.1:36943_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_en.txt to /configs/c1.AUTOCREATED/lang/stopwords_en.txt
   [junit4]   2> 202621 INFO  (OverseerThreadFactory-1826-thread-1-processing-n:127.0.0.1:36943_solr) [n:127.0.0.1:36943_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_lv.txt to /configs/c1.AUTOCREATED/lang/stopwords_lv.txt
   [junit4]   2> 202625 INFO  (OverseerThreadFactory-1826-thread-1-processing-n:127.0.0.1:36943_solr) [n:127.0.0.1:36943_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_th.txt to /configs/c1.AUTOCREATED/lang/stopwords_th.txt
   [junit4]   2> 202627 INFO  (OverseerThreadFactory-1826-thread-1-processing-n:127.0.0.1:36943_solr) [n:127.0.0.1:36943_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_ro.txt to /configs/c1.AUTOCREATED/lang/stopwords_ro.txt
   [junit4]   2> 202630 INFO  (OverseerThreadFactory-1826-thread-1-processing-n:127.0.0.1:36943_solr) [n:127.0.0.1:36943_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_fr.txt to /configs/c1.AUTOCREATED/lang/stopwords_fr.txt
   [junit4]   2> 202632 INFO  (OverseerThreadFactory-1826-thread-1-processing-n:127.0.0.1:36943_solr) [n:127.0.0.1:36943_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_sv.txt to /configs/c1.AUTOCREATED/lang/stopwords_sv.txt
   [junit4]   2> 202634 INFO  (OverseerThreadFactory-1826-thread-1-processing-n:127.0.0.1:36943_solr) [n:127.0.0.1:36943_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_fi.txt to /configs/c1.AUTOCREATED/lang/stopwords_fi.txt
   [junit4]   2> 202637 INFO  (OverseerThreadFactory-1826-thread-1-processing-n:127.0.0.1:36943_solr) [n:127.0.0.1:36943_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/hyphenations_ga.txt to /configs/c1.AUTOCREATED/lang/hyphenations_ga.txt
   [junit4]   2> 202639 INFO  (OverseerThreadFactory-1826-thread-1-processing-n:127.0.0.1:36943_solr) [n:127.0.0.1:36943_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_gl.txt to /configs/c1.AUTOCREATED/lang/stopwords_gl.txt
   [junit4]   2> 202642 INFO  (OverseerThreadFactory-1826-thread-1-processing-n:127.0.0.1:36943_solr) [n:127.0.0.1:36943_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/synonyms.txt to /configs/c1.AUTOCREATED/synonyms.txt
   [junit4]   2> 202645 INFO  (OverseerThreadFactory-1826-thread-1-processing-n:127.0.0.1:36943_solr) [n:127.0.0.1:36943_solr     ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/stopwords.txt to /configs/c1.AUTOCREATED/stopwords.txt
   [junit4]   2> 202759 INFO  (OverseerStateUpdate-72079963249639428-127.0.0.1:36943_solr-n_0000000000) [n:127.0.0.1:36943_solr     ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"c1",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"c1_shard1_replica_n1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:36943/solr",
   [junit4]   2>   "node_name":"127.0.0.1:36943_solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 202963 INFO  (qtp376837928-3250) [n:127.0.0.1:36943_solr    x:c1_shard1_replica_n1 ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node2&collection.configName=c1.AUTOCREATED&newCollection=true&name=c1_shard1_replica_n1&action=CREATE&numShards=1&collection=c1&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 202963 INFO  (qtp376837928-3250) [n:127.0.0.1:36943_solr    x:c1_shard1_replica_n1 ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 1000 transient cores
   [junit4]   2> 202976 INFO  (qtp376837928-3250) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.0.0
   [junit4]   2> 203000 INFO  (qtp376837928-3250) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.c.CoreContainer Creating SolrCore 'c1_shard1_replica_n1' using configuration from configset c1.AUTOCREATED, trusted=true
   [junit4]   2> 203006 INFO  (qtp376837928-3250) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.s.IndexSchema Schema name=default-config
   [junit4]   2> 203208 INFO  (qtp376837928-3250) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.s.IndexSchema Loaded schema default-config/1.6 with uniqueid field id
   [junit4]   2> 203210 INFO  (qtp376837928-3250) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_36943.solr.core.c1.shard1.replica_n1' (registry 'solr.core.c1.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@148ebc3e
   [junit4]   2> 203210 INFO  (qtp376837928-3250) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.c.SolrCore [[c1_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.servlet.TestRequestRateLimiter_D075C04A1EF5F5CE-001/tempDir-001/node1/c1_shard1_replica_n1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.servlet.TestRequestRateLimiter_D075C04A1EF5F5CE-001/tempDir-001/node1/c1_shard1_replica_n1/data/]
   [junit4]   2> 203292 INFO  (qtp376837928-3250) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 203292 INFO  (qtp376837928-3250) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 203294 INFO  (qtp376837928-3250) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.CommitTracker Hard AutoCommit: if uncommitted for 15000ms; 
   [junit4]   2> 203294 INFO  (qtp376837928-3250) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 203297 INFO  (qtp376837928-3250) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/c1.AUTOCREATED
   [junit4]   2> 203298 INFO  (qtp376837928-3250) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/c1.AUTOCREATED
   [junit4]   2> 203298 INFO  (qtp376837928-3250) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/c1.AUTOCREATED/managed-schema
   [junit4]   2> 203299 INFO  (qtp376837928-3250) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
   [junit4]   2> 203299 INFO  (qtp376837928-3250) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.h.c.SpellCheckComponent Initializing spell checkers
   [junit4]   2> 203299 INFO  (qtp376837928-3250) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.s.DirectSolrSpellChecker init: {name=default, field=_text_, classname=solr.DirectSolrSpellChecker, distanceMeasure=internal, accuracy=0.5, maxEdits=2, minPrefix=1, maxInspections=5, minQueryLength=4, maxQueryFrequency=0.01}
   [junit4]   2> 203300 INFO  (qtp376837928-3250) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 ms
   [junit4]   2> 203301 INFO  (qtp376837928-3250) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1674715186747932672
   [junit4]   2> 203303 INFO  (searcherExecutor-1838-thread-1-processing-n:127.0.0.1:36943_solr x:c1_shard1_replica_n1 c:c1 s:shard1 r:core_node2) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.c.QuerySenderListener QuerySenderListener done.
   [junit4]   2> 203304 INFO  (searcherExecutor-1838-thread-1-processing-n:127.0.0.1:36943_solr x:c1_shard1_replica_n1 c:c1 s:shard1 r:core_node2) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
   [junit4]   2> 203306 INFO  (searcherExecutor-1838-thread-1-processing-n:127.0.0.1:36943_solr x:c1_shard1_replica_n1 c:c1 s:shard1 r:core_node2) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.c.SolrCore [c1_shard1_replica_n1]  Registered new searcher autowarm time: 0 ms
   [junit4]   2> 203308 INFO  (qtp376837928-3250) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.c.ZkShardTerms Successful update of terms at /collections/c1/terms/shard1 to Terms{values={core_node2=0}, version=0}
   [junit4]   2> 203308 INFO  (qtp376837928-3250) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/c1/leaders/shard1
   [junit4]   2> 203313 INFO  (qtp376837928-3250) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 203313 INFO  (qtp376837928-3250) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 203313 INFO  (qtp376837928-3250) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:36943/solr/c1_shard1_replica_n1/
   [junit4]   2> 203314 INFO  (qtp376837928-3250) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 203314 INFO  (qtp376837928-3250) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.c.SyncStrategy http://127.0.0.1:36943/solr/c1_shard1_replica_n1/ has no replicas
   [junit4]   2> 203314 INFO  (qtp376837928-3250) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/c1/leaders/shard1/leader after winning as /collections/c1/leader_elect/shard1/election/72079963249639428-core_node2-n_0000000000
   [junit4]   2> 203317 INFO  (qtp376837928-3250) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:36943/solr/c1_shard1_replica_n1/ shard1
   [junit4]   2> 203420 INFO  (zkCallback-1820-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/c1/state.json] for collection [c1] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 203420 INFO  (zkCallback-1820-thread-2) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/c1/state.json] for collection [c1] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 203422 INFO  (qtp376837928-3250) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 203424 INFO  (qtp376837928-3250) [n:127.0.0.1:36943_solr     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node2&collection.configName=c1.AUTOCREATED&newCollection=true&name=c1_shard1_replica_n1&action=CREATE&numShards=1&collection=c1&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=461
   [junit4]   2> 203427 INFO  (qtp376837928-3249) [n:127.0.0.1:36943_solr     ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
   [junit4]   2> 203524 INFO  (zkCallback-1820-thread-2) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/c1/state.json] for collection [c1] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 203524 INFO  (zkCallback-1820-thread-3) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/c1/state.json] for collection [c1] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 203524 INFO  (zkCallback-1820-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/c1/state.json] for collection [c1] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 203525 INFO  (qtp376837928-3249) [n:127.0.0.1:36943_solr     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={pullReplicas=0&name=c1&nrtReplicas=1&action=CREATE&numShards=1&tlogReplicas=0&wt=javabin&version=2} status=0 QTime=1003
   [junit4]   2> 203526 INFO  (TEST-TestRequestRateLimiter.testConcurrentQueries-seed#[D075C04A1EF5F5CE]) [     ] o.a.s.c.MiniSolrCloudCluster waitForActiveCollection: c1
   [junit4]   2> 203653 INFO  (qtp376837928-3248) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.s.ManagedIndexSchema Persisted managed schema version 1  at /configs/c1.AUTOCREATED/managed-schema
   [junit4]   2> 203653 INFO  (zkCallback-1820-thread-4) [     ] o.a.s.s.ZkIndexSchemaReader A schema change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/configs/c1.AUTOCREATED/managed-schema, has occurred - updating schema from ZooKeeper ...
   [junit4]   2> 203653 INFO  (zkCallback-1820-thread-4) [     ] o.a.s.s.ZkIndexSchemaReader Retrieved schema version 1 from Zookeeper
   [junit4]   2> 203656 INFO  (qtp376837928-3248) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.c.ZkShardTerms Successful update of terms at /collections/c1/terms/shard1 to Terms{values={core_node2=1}, version=1}
   [junit4]   2> 203656 INFO  (qtp376837928-3248) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[0 (1674715187117031424)]} 0 127
   [junit4]   2> 203659 INFO  (qtp376837928-3250) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[1 (1674715187122274304)]} 0 2
   [junit4]   2> 203660 INFO  (qtp376837928-3249) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[2 (1674715187124371456)]} 0 0
   [junit4]   2> 203660 INFO  (zkCallback-1820-thread-4) [     ] o.a.s.s.IndexSchema Schema name=default-config
   [junit4]   2> 203661 INFO  (qtp376837928-3248) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[3 (1674715187125420032)]} 0 0
   [junit4]   2> 203662 INFO  (qtp376837928-3250) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[4 (1674715187126468608)]} 0 0
   [junit4]   2> 203663 INFO  (qtp376837928-3249) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[5 (1674715187127517184)]} 0 0
   [junit4]   2> 203663 INFO  (qtp376837928-3248) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[6 (1674715187127517185)]} 0 0
   [junit4]   2> 203664 INFO  (qtp376837928-3250) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[7 (1674715187128565760)]} 0 0
   [junit4]   2> 203665 INFO  (qtp376837928-3249) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[8 (1674715187129614336)]} 0 0
   [junit4]   2> 203666 INFO  (qtp376837928-3248) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[9 (1674715187130662912)]} 0 0
   [junit4]   2> 203667 INFO  (qtp376837928-3250) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[10 (1674715187131711488)]} 0 0
   [junit4]   2> 203668 INFO  (qtp376837928-3249) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[11 (1674715187131711489)]} 0 0
   [junit4]   2> 203668 INFO  (qtp376837928-3248) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[12 (1674715187132760064)]} 0 0
   [junit4]   2> 203669 INFO  (qtp376837928-3250) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[13 (1674715187133808640)]} 0 0
   [junit4]   2> 203670 INFO  (qtp376837928-3249) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[14 (1674715187134857216)]} 0 0
   [junit4]   2> 203671 INFO  (qtp376837928-3248) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[15 (1674715187135905792)]} 0 0
   [junit4]   2> 203672 INFO  (qtp376837928-3250) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[16 (1674715187136954368)]} 0 0
   [junit4]   2> 203673 INFO  (qtp376837928-3249) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[17 (1674715187138002944)]} 0 0
   [junit4]   2> 203674 INFO  (qtp376837928-3248) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[18 (1674715187139051520)]} 0 0
   [junit4]   2> 203675 INFO  (qtp376837928-3250) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[19 (1674715187140100096)]} 0 0
   [junit4]   2> 203676 INFO  (qtp376837928-3249) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[20 (1674715187141148672)]} 0 0
   [junit4]   2> 203677 INFO  (qtp376837928-3248) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[21 (1674715187141148673)]} 0 0
   [junit4]   2> 203677 INFO  (qtp376837928-3250) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[22 (1674715187142197248)]} 0 0
   [junit4]   2> 203678 INFO  (qtp376837928-3249) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[23 (1674715187143245824)]} 0 0
   [junit4]   2> 203679 INFO  (qtp376837928-3248) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[24 (1674715187144294400)]} 0 0
   [junit4]   2> 203680 INFO  (qtp376837928-3250) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[25 (1674715187145342976)]} 0 0
   [junit4]   2> 203681 INFO  (qtp376837928-3249) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[26 (1674715187146391552)]} 0 0
   [junit4]   2> 203682 INFO  (qtp376837928-3248) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[27 (1674715187147440128)]} 0 0
   [junit4]   2> 203683 INFO  (qtp376837928-3250) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[28 (1674715187147440129)]} 0 0
   [junit4]   2> 203683 INFO  (qtp376837928-3249) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[29 (1674715187148488704)]} 0 0
   [junit4]   2> 203684 INFO  (qtp376837928-3248) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[30 (1674715187149537280)]} 0 0
   [junit4]   2> 203685 INFO  (qtp376837928-3250) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[31 (1674715187150585856)]} 0 0
   [junit4]   2> 203686 INFO  (qtp376837928-3249) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[32 (1674715187151634432)]} 0 0
   [junit4]   2> 203687 INFO  (qtp376837928-3248) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[33 (1674715187152683008)]} 0 0
   [junit4]   2> 203688 INFO  (qtp376837928-3250) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[34 (1674715187153731584)]} 0 0
   [junit4]   2> 203689 INFO  (qtp376837928-3249) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[35 (1674715187154780160)]} 0 0
   [junit4]   2> 203690 INFO  (qtp376837928-3248) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[36 (1674715187154780161)]} 0 0
   [junit4]   2> 203690 INFO  (qtp376837928-3250) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[37 (1674715187155828736)]} 0 0
   [junit4]   2> 203691 INFO  (qtp376837928-3249) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[38 (1674715187156877312)]} 0 0
   [junit4]   2> 203692 INFO  (qtp376837928-3248) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[39 (1674715187157925888)]} 0 0
   [junit4]   2> 203693 INFO  (qtp376837928-3250) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[40 (1674715187158974464)]} 0 0
   [junit4]   2> 203694 INFO  (qtp376837928-3249) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[41 (1674715187160023040)]} 0 0
   [junit4]   2> 203695 INFO  (qtp376837928-3248) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[42 (1674715187161071616)]} 0 0
   [junit4]   2> 203696 INFO  (qtp376837928-3250) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[43 (1674715187161071617)]} 0 0
   [junit4]   2> 203696 INFO  (qtp376837928-3249) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[44 (1674715187162120192)]} 0 0
   [junit4]   2> 203697 INFO  (qtp376837928-3248) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[45 (1674715187163168768)]} 0 0
   [junit4]   2> 203698 INFO  (qtp376837928-3250) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[46 (1674715187164217344)]} 0 0
   [junit4]   2> 203699 INFO  (qtp376837928-3249) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[47 (1674715187165265920)]} 0 0
   [junit4]   2> 203700 INFO  (qtp376837928-3248) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[48 (1674715187166314496)]} 0 0
   [junit4]   2> 203701 INFO  (qtp376837928-3250) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[49 (1674715187167363072)]} 0 0
   [junit4]   2> 203702 INFO  (qtp376837928-3249) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[50 (1674715187168411648)]} 0 0
   [junit4]   2> 203703 INFO  (qtp376837928-3248) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[51 (1674715187169460224)]} 0 0
   [junit4]   2> 203704 INFO  (qtp376837928-3250) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[52 (1674715187170508800)]} 0 0
   [junit4]   2> 203705 INFO  (qtp376837928-3249) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[53 (1674715187170508801)]} 0 0
   [junit4]   2> 203705 INFO  (qtp376837928-3248) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[54 (1674715187171557376)]} 0 0
   [junit4]   2> 203706 INFO  (qtp376837928-3250) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[55 (1674715187172605952)]} 0 0
   [junit4]   2> 203707 INFO  (qtp376837928-3249) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[56 (1674715187173654528)]} 0 0
   [junit4]   2> 203708 INFO  (qtp376837928-3248) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[57 (1674715187174703104)]} 0 0
   [junit4]   2> 203709 INFO  (qtp376837928-3250) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[58 (1674715187175751680)]} 0 0
   [junit4]   2> 203710 INFO  (qtp376837928-3249) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[59 (1674715187175751681)]} 0 0
   [junit4]   2> 203710 INFO  (qtp376837928-3248) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[60 (1674715187176800256)]} 0 0
   [junit4]   2> 203711 INFO  (qtp376837928-3250) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[61 (1674715187177848832)]} 0 0
   [junit4]   2> 203712 INFO  (qtp376837928-3249) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[62 (1674715187178897408)]} 0 0
   [junit4]   2> 203713 INFO  (qtp376837928-3248) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[63 (1674715187179945984)]} 0 0
   [junit4]   2> 203714 INFO  (qtp376837928-3250) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[64 (1674715187180994560)]} 0 0
   [junit4]   2> 203715 INFO  (qtp376837928-3249) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[65 (1674715187180994561)]} 0 0
   [junit4]   2> 203715 INFO  (qtp376837928-3248) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[66 (1674715187182043136)]} 0 0
   [junit4]   2> 203716 INFO  (qtp376837928-3250) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[67 (1674715187183091712)]} 0 0
   [junit4]   2> 203717 INFO  (qtp376837928-3249) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[68 (1674715187184140288)]} 0 0
   [junit4]   2> 203718 INFO  (qtp376837928-3248) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[69 (1674715187185188864)]} 0 0
   [junit4]   2> 203719 INFO  (qtp376837928-3250) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[70 (1674715187186237440)]} 0 0
   [junit4]   2> 203720 INFO  (qtp376837928-3249) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[71 (1674715187187286016)]} 0 0
   [junit4]   2> 203721 INFO  (qtp376837928-3248) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[72 (1674715187187286017)]} 0 0
   [junit4]   2> 203721 INFO  (qtp376837928-3250) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[73 (1674715187188334592)]} 0 0
   [junit4]   2> 203722 INFO  (qtp376837928-3249) [n:127.0.0.1:36943_solr c:c1 s:shard1 r:core_node2 x:c1_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [c1_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[7

[...truncated too long message...]

     ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty tag=null
   [junit4]   2> 6324520 INFO  (jetty-closer-30659-thread-1) [     ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@40da9b6d: rootName = solr_37311, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty/com.codahale.metrics.MetricRegistry@18a4ffe5
   [junit4]   2> 6324520 INFO  (jetty-closer-30659-thread-1) [     ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster tag=null
   [junit4]   2> 6324638 INFO  (jetty-closer-30659-thread-1) [     ] o.e.j.s.AbstractConnector Stopped ServerConnector@9be9e3d{ssl, (ssl, alpn, http/1.1, h2)}{127.0.0.1:0}
   [junit4]   2> 6324638 INFO  (jetty-closer-30659-thread-1) [     ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@3586f89a{/solr,null,UNAVAILABLE}
   [junit4]   2> 6324639 INFO  (jetty-closer-30659-thread-1) [     ] o.e.j.s.session node0 Stopped scavenging
   [junit4]   2> 6324639 INFO  (SUITE-TestLocalFSCloudBackupRestore-seed#[D075C04A1EF5F5CE]-worker) [     ] o.a.s.c.ZkTestServer Shutting down ZkTestServer.
   [junit4]   2> 6324742 WARN  (SUITE-TestLocalFSCloudBackupRestore-seed#[D075C04A1EF5F5CE]-worker-SendThread(127.0.0.1:45413)) [     ] o.a.z.ClientCnxn An exception was thrown while closing send thread for session 0x10014b53ef00001.
   [junit4]   2>           => EndOfStreamException: Unable to read additional data from server sessionid 0x10014b53ef00001, likely server has closed socket
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:75)
   [junit4]   2> org.apache.zookeeper.ClientCnxn$EndOfStreamException: Unable to read additional data from server sessionid 0x10014b53ef00001, likely server has closed socket
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:75) ~[zookeeper-3.6.1.jar:3.6.1]
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:348) ~[zookeeper-3.6.1.jar:3.6.1]
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1262) [zookeeper-3.6.1.jar:3.6.1]
   [junit4]   2> 6324844 WARN  (ZkTestServer Run Thread) [     ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	10	/solr/collections/backuprestore_testok/terms/shard1_0
   [junit4]   2> 	7	/solr/collections/backuprestore_testfailure/terms/shard1
   [junit4]   2> 	7	/solr/collections/backuprestore_testfailure/terms/shard2
   [junit4]   2> 	7	/solr/collections/backuprestore_testok/terms/shard2
   [junit4]   2> 	6	/solr/collections/backuprestore_testok_restored/terms/shard1_1
   [junit4]   2> 	6	/solr/collections/backuprestore_testok_restored/terms/shard2
   [junit4]   2> 	6	/solr/collections/backuprestore_testok/terms/shard1_1
   [junit4]   2> 	6	/solr/collections/backuprestore_testok_restored/terms/shard1_0
   [junit4]   2> 	5	/solr/collections/backuprestore_testok/terms/shard1
   [junit4]   2> 	3	/solr/aliases.json
   [junit4]   2> 	2	/solr/collections/backuprestore_testfailure/collectionprops.json
   [junit4]   2> 	2	/solr/packages.json
   [junit4]   2> 	2	/solr/security.json
   [junit4]   2> 	2	/solr/collections/backuprestore_testok/collectionprops.json
   [junit4]   2> 	2	/solr/configs/conf1
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	153	/solr/collections/backuprestore_testok/state.json
   [junit4]   2> 	91	/solr/collections/backuprestore_testok_restored/state.json
   [junit4]   2> 	47	/solr/collections/backuprestore_testfailure/state.json
   [junit4]   2> 	3	/solr/clusterprops.json
   [junit4]   2> 	3	/solr/collections/backuprestore_testfailure_restored/state.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	158	/solr/overseer/queue
   [junit4]   2> 	23	/solr/overseer/collection-queue-work
   [junit4]   2> 	18	/solr/collections
   [junit4]   2> 	11	/solr/live_nodes
   [junit4]   2> 
   [junit4]   2> 6324872 INFO  (SUITE-TestLocalFSCloudBackupRestore-seed#[D075C04A1EF5F5CE]-worker) [     ] o.a.s.c.ZkTestServer waitForServerDown: 127.0.0.1:45413
   [junit4]   2> 6324872 INFO  (SUITE-TestLocalFSCloudBackupRestore-seed#[D075C04A1EF5F5CE]-worker) [     ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:45413
   [junit4]   2> 6324872 INFO  (SUITE-TestLocalFSCloudBackupRestore-seed#[D075C04A1EF5F5CE]-worker) [     ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 45413
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.api.collections.TestLocalFSCloudBackupRestore_D075C04A1EF5F5CE-001
   [junit4]   2> NOTE: test params are: codec=Lucene86, sim=Asserting(RandomSimilarity(queryNorm=false): {}), locale=gv, timezone=America/Dawson
   [junit4]   2> NOTE: Linux 4.15.0-112-generic amd64/Oracle Corporation 11.0.4 (64-bit)/cpus=8,threads=1,free=117554080,total=469762048
   [junit4]   2> NOTE: All tests run in this JVM: [SolrCLIZkUtilsTest, TestPullReplicaErrorHandling, ShardRoutingTest, TestDefaultStatsCache, LeaderVoteWaitTimeoutTest, EchoParamsTest, DistributedQueueTest, TestHashQParserPlugin, SolrMetricManagerTest, AuthToolTest, TestCoreBackup, TestCloudJSONFacetJoinDomain, TestRequestRateLimiter, OverseerTest, TestPushWriter, TestExactStatsCacheCloud, TestSolrJ, TestJsonFacetErrors, RequestLoggingTest, ByteBuffersDirectoryFactoryTest, UninvertDocValuesMergePolicyTest, CloudExitableDirectoryReaderTest, HdfsTlogReplayBufferedWhileIndexingTest, TermVectorComponentTest, ClassificationUpdateProcessorIntegrationTest, LeaderElectionContextKeyTest, TestManagedResourceStorage, TestExpandComponent, TestFiltering, DistributedTermsComponentTest, OutOfBoxZkACLAndCredentialsProvidersTest, AlternateDirectoryTest, TestAddFieldRealTimeGet, HdfsRestartWhileUpdatingTest, TestSearchPerf, TestCSVLoader, ZkCLITest, RankFieldTest, TestOmitPositions, CdcrBidirectionalTest, TestReplicaProperties, TestReversedWildcardFilterFactory, SimpleCollectionCreateDeleteTest, TestLegacyField, TestRandomFlRTGCloud, TestFieldCache, FileUtilsTest, TestAuthenticationFramework, TestSolrConfigHandlerCloud, RegexBytesRefFilterTest, TestNestedUpdateProcessor, ZkStateReaderTest, TestBulkSchemaConcurrent, PeerSyncWithLeaderTest, TestDirectoryFactory, TestManagedSchema, TestValueSourceCache, DocValuesMissingTest, TestSubQueryTransformerCrossCore, DistributedQueryElevationComponentTest, AssignBackwardCompatibilityTest, EmbeddedSolrNoSerializeTest, DeleteInactiveReplicaTest, TolerantUpdateProcessorTest, OverseerModifyCollectionTest, MoreLikeThisComponentTest, JvmMetricsTest, TestSchemaManager, CSVRequestHandlerTest, TestUninvertingReader, LargeFieldTest, TestTolerantUpdateProcessorRandomCloud, TestFacetMethods, ParsingFieldUpdateProcessorsTest, SolrRrdBackendFactoryTest, TestEmbeddedSolrServerSchemaAPI, HLLUtilTest, DistributedFacetPivotSmallAdvancedTest, QueryEqualityTest, AsyncCallRequestStatusResponseTest, PreAnalyzedFieldManagedSchemaCloudTest, TestCloudRecovery2, MultiDestinationAuditLoggerTest, SpellCheckCollatorWithCollapseTest, TestChildDocTransformerHierarchy, CoreSorterTest, ShardsWhitelistTest, RecoveryAfterSoftCommitTest, TestExtendedDismaxParser, TestManagedSynonymGraphFilterFactory, TestCloudNestedDocsSort, TestLRUStatsCacheCloud, TestSmileRequest, UniqFieldsUpdateProcessorFactoryTest, FileBasedSpellCheckerTest, ActionThrottleTest, DateMathParserTest, TestRequestForwarding, SolrIndexSplitterTest, TestSystemCollAutoCreate, DebugComponentTest, MetricUtilsTest, TestLegacyBM25SimilarityFactory, TestRestoreCore, HdfsBasicDistributedZkTest, CdcrReplicationHandlerTest, TriLevelCompositeIdRoutingTest, TestMissingGroups, TestInPlaceUpdatesRequiredField, TestCustomSort, UpdateLogTest, TestTlogReplayVsRecovery, TestTestInjection, SplitShardTest, TestTrie, MigrateRouteKeyTest, TestFieldCacheSortRandom, TestClassicSimilarityFactory, TestStressCloudBlindAtomicUpdates, TestCoreDiscovery, TestConfigSetProperties, TestDownShardTolerantSearch, TestHdfsUpdateLog, ChaosMonkeySafeLeaderWithPullReplicasTest, BlockDirectoryTest, TestConfigsApi, TestAuthorizationFramework, TestRuleBasedAuthorizationWithKerberos, CacheHeaderTest, ClusterStateUpdateTest, CloudMLTQParserTest, TestXIncludeConfig, TestWriterPerf, AnalysisAfterCoreReloadTest, SuggestComponentContextFilterQueryTest, DistributedFacetExistsSmallTest, SchemaApiFailureTest, DistributedFacetSimpleRefinementLongTailTest, TestRTGBase, CollectionsAPISolrJTest, JWTAuthPluginTest, TestSolrDeletionPolicy2, RestartWhileUpdatingTest, RouteFieldTest, SSLMigrationTest, SaslZkACLProviderTest, ShardRoutingCustomTest, TestCloudDeleteByQuery, TestCloudInspectUtil, TestCloudPhrasesIdentificationComponent, TestCloudPivotFacet, TestCloudPseudoReturnFields, TestCloudRecovery, TestConfigSetsAPI, TestConfigSetsAPIExclusivity, TestConfigSetsAPIZkFailure, TestDeleteCollectionOnDownNodes, TestDistribDocBasedVersion, TestDistributedMap, TestLocalStatsCacheCloud, TestLockTree, TestMiniSolrCloudClusterSSL, TestOnReconnectListenerSupport, TestPrepRecovery, TestPullReplica, TestShortCircuitedRequests, TestSizeLimitedDistributedMap, TestSkipOverseerOperations, TestSolrCloudWithDelegationTokens, TestSolrCloudWithKerberosAlt, TestSolrCloudWithSecureImpersonation, ZkNodePropsTest, ZkShardTermsTest, ZkSolrClientTest, CollectionsAPIAsyncDistributedZkTest, CollectionsAPIDistributedZkTest, ConcurrentDeleteAndCreateCollectionTest, CustomCollectionTest, HdfsCollectionsAPIDistributedZkTest, TestCollectionsAPIViaSolrCloudCluster, TestHdfsCloudBackupRestore, TestLocalFSCloudBackupRestore]
   [junit4] Completed [509/864 (3!)] on J0 in 27.61s, 2 tests, 1 failure <<< FAILURES!

[...truncated 53104 lines...]

[JENKINS] Lucene » Lucene-Solr-NightlyTests-master - Build # 4 - Still Unstable!

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://ci-builds.apache.org/job/Lucene/job/Lucene-Solr-NightlyTests-master/4/

2 tests failed.
FAILED:  org.apache.solr.cloud.DocValuesNotIndexedTest.testGroupingDVOnlySortLast

Error Message:
Unexpected number of elements in the group for 'intGSL' size: '8' GroupValue: 'null' rsp: {responseHeader={zkConnected=true, status=0, QTime=6, params={q=*:*, group.sort=id asc, _stateVer_=dv_coll:4, group.limit=100, rows=100, wt=javabin, version=2, group.field=intGSL, group=true}}, grouped={intGSL={matches=50, groups=[{groupValue=127893651, doclist={numFound=7,numFoundExact=true,start=0,maxScore=1.0,docs=[SolrDocument{id=0, intGSL=127893651, longGSL=1312574611551989823, doubleGSL=4390.351806523658, floatGSL=2273.3347, dateGSL=Tue Sep 17 08:01:51 AKDT 107784622, stringGSL=base_string_447973__00005883, boolGSL=true, sortableGSL=base_string_772653__00009563, _version_=1674636562951831552, _root_=0}, SolrDocument{id=1, intGSL=127893651, longGSL=1312574611551989823, doubleGSL=4390.351806523658, floatGSL=2273.3347, dateGSL=Tue Sep 17 08:01:51 AKDT 107784622, stringGSL=base_string_447973__00005883, boolGSL=false, sortableGSL=base_string_772653__00009563, _version_=1674636562949734400, _root_=1}, SolrDocument{id=2, intGSL=127893651, longGSL=1312574611551989823, doubleGSL=4390.351806523658, floatGSL=2273.3347, dateGSL=Tue Sep 17 08:01:51 AKDT 107784622, stringGSL=base_string_447973__00005883, boolGSL=true, sortableGSL=base_string_772653__00009563, _version_=1674636562948685824, _root_=2}, SolrDocument{id=3, intGSL=127893651, longGSL=1312574611551989823, doubleGSL=4390.351806523658, floatGSL=2273.3347, dateGSL=Tue Sep 17 08:01:51 AKDT 107784622, stringGSL=base_string_447973__00005883, boolGSL=false, sortableGSL=base_string_772653__00009563, _version_=1674636562950782976, _root_=3}, SolrDocument{id=4, intGSL=127893651, longGSL=1312574611551989823, doubleGSL=4390.351806523658, floatGSL=2273.3347, dateGSL=Tue Sep 17 08:01:51 AKDT 107784622, stringGSL=base_string_447973__00005883, boolGSL=true, sortableGSL=base_string_772653__00009563, _version_=1674636562952880128, _root_=4}, SolrDocument{id=5, intGSL=127893651, longGSL=1312574611551989823, doubleGSL=4390.351806523658, floatGSL=2273.3347, dateGSL=Tue Sep 17 08:01:51 AKDT 107784622, stringGSL=base_string_447973__00005883, boolGSL=false, sortableGSL=base_string_772653__00009563, _version_=1674636562950782976, _root_=5}, SolrDocument{id=6, intGSL=127893651, longGSL=1312574611551989823, doubleGSL=4390.351806523658, floatGSL=2273.3347, dateGSL=Tue Sep 17 08:01:51 AKDT 107784622, stringGSL=base_string_447973__00005883, boolGSL=true, sortableGSL=base_string_772653__00009563, _version_=1674636562950782977, _root_=6}]}}, {groupValue=127893821, doclist={numFound=7,numFoundExact=true,start=0,maxScore=1.0,docs=[SolrDocument{id=10, intGSL=127893821, longGSL=1312574611551992306, doubleGSL=9048.351806523657, floatGSL=4402.3345, dateGSL=Tue Sep 17 08:01:58 AKDT 107784622, stringGSL=base_string_447973__00015653, boolGSL=true, sortableGSL=base_string_772653__00010968, _version_=1674636562951831552, _root_=10}, SolrDocument{id=11, intGSL=127893821, longGSL=1312574611551992306, doubleGSL=9048.351806523657, floatGSL=4402.3345, dateGSL=Tue Sep 17 08:01:58 AKDT 107784622, stringGSL=base_string_447973__00015653, boolGSL=false, sortableGSL=base_string_772653__00010968, _version_=1674636562951831553, _root_=11}, SolrDocument{id=12, intGSL=127893821, longGSL=1312574611551992306, doubleGSL=9048.351806523657, floatGSL=4402.3345, dateGSL=Tue Sep 17 08:01:58 AKDT 107784622, stringGSL=base_string_447973__00015653, boolGSL=true, sortableGSL=base_string_772653__00010968, _version_=1674636562952880130, _root_=12}, SolrDocument{id=13, intGSL=127893821, longGSL=1312574611551992306, doubleGSL=9048.351806523657, floatGSL=4402.3345, dateGSL=Tue Sep 17 08:01:58 AKDT 107784622, stringGSL=base_string_447973__00015653, boolGSL=false, sortableGSL=base_string_772653__00010968, _version_=1674636562951831554, _root_=13}, SolrDocument{id=7, intGSL=127893821, longGSL=1312574611551992306, doubleGSL=9048.351806523657, floatGSL=4402.3345, dateGSL=Tue Sep 17 08:01:58 AKDT 107784622, stringGSL=base_string_447973__00015653, boolGSL=false, sortableGSL=base_string_772653__00010968, _version_=1674636562950782978, _root_=7}, SolrDocument{id=8, intGSL=127893821, longGSL=1312574611551992306, doubleGSL=9048.351806523657, floatGSL=4402.3345, dateGSL=Tue Sep 17 08:01:58 AKDT 107784622, stringGSL=base_string_447973__00015653, boolGSL=true, sortableGSL=base_string_772653__00010968, _version_=1674636562950782976, _root_=8}, SolrDocument{id=9, intGSL=127893821, longGSL=1312574611551992306, doubleGSL=9048.351806523657, floatGSL=4402.3345, dateGSL=Tue Sep 17 08:01:58 AKDT 107784622, stringGSL=base_string_447973__00015653, boolGSL=false, sortableGSL=base_string_772653__00010968, _version_=1674636562950782979, _root_=9}]}}, {groupValue=null, doclist={numFound=8,numFoundExact=true,start=0,maxScore=1.0,docs=[SolrDocument{id=10000, _version_=1674636562949734400, _root_=10000}, SolrDocument{id=10005, _version_=1674636562952880129, _root_=10005}, SolrDocument{id=10010, _version_=1674636562951831552, _root_=10010}, SolrDocument{id=10015, _version_=1674636562951831556, _root_=10015}, SolrDocument{id=10020, _version_=1674636562951831557, _root_=10020}, SolrDocument{id=10025, _version_=1674636562951831554, _root_=10025}, SolrDocument{id=10035, _version_=1674636562951831558, _root_=10035}, SolrDocument{id=10045, _version_=1674636562951831560, _root_=10045}]}}, {groupValue=127894535, doclist={numFound=7,numFoundExact=true,start=0,maxScore=1.0,docs=[SolrDocument{id=14, intGSL=127894535, longGSL=1312574611551997559, doubleGSL=11174.351806523657, floatGSL=4725.3345, dateGSL=Tue Sep 17 08:02:07 AKDT 107784622, stringGSL=base_string_447973__00023825, boolGSL=true, sortableGSL=base_string_772653__00020865, _version_=1674636562951831555, _root_=14}, SolrDocument{id=15, intGSL=127894535, longGSL=1312574611551997559, doubleGSL=11174.351806523657, floatGSL=4725.3345, dateGSL=Tue Sep 17 08:02:07 AKDT 107784622, stringGSL=base_string_447973__00023825, boolGSL=false, sortableGSL=base_string_772653__00020865, _version_=1674636562952880131, _root_=15}, SolrDocument{id=16, intGSL=127894535, longGSL=1312574611551997559, doubleGSL=11174.351806523657, floatGSL=4725.3345, dateGSL=Tue Sep 17 08:02:07 AKDT 107784622, stringGSL=base_string_447973__00023825, boolGSL=true, sortableGSL=base_string_772653__00020865, _version_=1674636562953928704, _root_=16}, SolrDocument{id=17, intGSL=127894535, longGSL=1312574611551997559, doubleGSL=11174.351806523657, floatGSL=4725.3345, dateGSL=Tue Sep 17 08:02:07 AKDT 107784622, stringGSL=base_string_447973__00023825, boolGSL=false, sortableGSL=base_string_772653__00020865, _version_=1674636562950782977, _root_=17}, SolrDocument{id=18, intGSL=127894535, longGSL=1312574611551997559, doubleGSL=11174.351806523657, floatGSL=4725.3345, dateGSL=Tue Sep 17 08:02:07 AKDT 107784622, stringGSL=base_string_447973__00023825, boolGSL=true, sortableGSL=base_string_772653__00020865, _version_=1674636562951831553, _root_=18}, SolrDocument{id=19, intGSL=127894535, longGSL=1312574611551997559, doubleGSL=11174.351806523657, floatGSL=4725.3345, dateGSL=Tue Sep 17 08:02:07 AKDT 107784622, stringGSL=base_string_447973__00023825, boolGSL=false, sortableGSL=base_string_772653__00020865, _version_=1674636562950782978, _root_=19}, SolrDocument{id=20, intGSL=127894535, longGSL=1312574611551997559, doubleGSL=11174.351806523657, floatGSL=4725.3345, dateGSL=Tue Sep 17 08:02:07 AKDT 107784622, stringGSL=base_string_447973__00023825, boolGSL=true, sortableGSL=base_string_772653__00020865, _version_=1674636562953928705, _root_=20}]}}, {groupValue=127898924, doclist={numFound=4,numFoundExact=true,start=0,maxScore=1.0,docs=[SolrDocument{id=22, intGSL=127898924, longGSL=1312574611552003071, doubleGSL=16074.351806523657, floatGSL=7109.3345, dateGSL=Tue Sep 17 08:02:14 AKDT 107784622, stringGSL=base_string_447973__00025921, boolGSL=true, sortableGSL=base_string_772653__00020905, _version_=1674636562953928706, _root_=22}, SolrDocument{id=24, intGSL=127898924, longGSL=1312574611552003071, doubleGSL=16074.351806523657, floatGSL=7109.3345, dateGSL=Tue Sep 17 08:02:14 AKDT 107784622, stringGSL=base_string_447973__00025921, boolGSL=true, sortableGSL=base_string_772653__00020905, _version_=1674636562951831558, _root_=24}, SolrDocument{id=26, intGSL=127898924, longGSL=1312574611552003071, doubleGSL=16074.351806523657, floatGSL=7109.3345, dateGSL=Tue Sep 17 08:02:14 AKDT 107784622, stringGSL=base_string_447973__00025921, boolGSL=true, sortableGSL=base_string_772653__00020905, _version_=1674636562951831555, _root_=26}, SolrDocument{id=27, intGSL=127898924, longGSL=1312574611552003071, doubleGSL=16074.351806523657, floatGSL=7109.3345, dateGSL=Tue Sep 17 08:02:14 AKDT 107784622, stringGSL=base_string_447973__00025921, boolGSL=false, sortableGSL=base_string_772653__00020905, _version_=1674636562951831559, _root_=27}]}}, {groupValue=127906471, doclist={numFound=5,numFoundExact=true,start=0,maxScore=1.0,docs=[SolrDocument{id=28, intGSL=127906471, longGSL=1312574611552009642, doubleGSL=19080.351806523657, floatGSL=10407.335, dateGSL=Tue Sep 17 08:02:19 AKDT 107784622, stringGSL=base_string_447973__00035445, boolGSL=true, sortableGSL=base_string_772653__00025749, _version_=1674636562951831560, _root_=28}, SolrDocument{id=30, intGSL=127906471, longGSL=1312574611552009642, doubleGSL=19080.351806523657, floatGSL=10407.335, dateGSL=Tue Sep 17 08:02:19 AKDT 107784622, stringGSL=base_string_447973__00035445, boolGSL=true, sortableGSL=base_string_772653__00025749, _version_=1674636562951831556, _root_=30}, SolrDocument{id=31, intGSL=127906471, longGSL=1312574611552009642, doubleGSL=19080.351806523657, floatGSL=10407.335, dateGSL=Tue Sep 17 08:02:19 AKDT 107784622, stringGSL=base_string_447973__00035445, boolGSL=false, sortableGSL=base_string_772653__00025749, _version_=1674636562951831557, _root_=31}, SolrDocument{id=32, intGSL=127906471, longGSL=1312574611552009642, doubleGSL=19080.351806523657, floatGSL=10407.335, dateGSL=Tue Sep 17 08:02:19 AKDT 107784622, stringGSL=base_string_447973__00035445, boolGSL=true, sortableGSL=base_string_772653__00025749, _version_=1674636562951831561, _root_=32}, SolrDocument{id=34, intGSL=127906471, longGSL=1312574611552009642, doubleGSL=19080.351806523657, floatGSL=10407.335, dateGSL=Tue Sep 17 08:02:19 AKDT 107784622, stringGSL=base_string_447973__00035445, boolGSL=true, sortableGSL=base_string_772653__00025749, _version_=1674636562953928707, _root_=34}]}}, {groupValue=127916419, doclist={numFound=6,numFoundExact=true,start=0,maxScore=1.0,docs=[SolrDocument{id=35, intGSL=127916419, longGSL=1312574611552010408, doubleGSL=27288.351806523657, floatGSL=15048.335, dateGSL=Tue Sep 17 08:02:28 AKDT 107784622, stringGSL=base_string_447973__00041606, boolGSL=false, sortableGSL=base_string_772653__00030178, _version_=1674636562953928708, _root_=35}, SolrDocument{id=36, intGSL=127916419, longGSL=1312574611552010408, doubleGSL=27288.351806523657, floatGSL=15048.335, dateGSL=Tue Sep 17 08:02:28 AKDT 107784622, stringGSL=base_string_447973__00041606, boolGSL=true, sortableGSL=base_string_772653__00030178, _version_=1674636562953928709, _root_=36}, SolrDocument{id=38, intGSL=127916419, longGSL=1312574611552010408, doubleGSL=27288.351806523657, floatGSL=15048.335, dateGSL=Tue Sep 17 08:02:28 AKDT 107784622, stringGSL=base_string_447973__00041606, boolGSL=true, sortableGSL=base_string_772653__00030178, _version_=1674636562952880128, _root_=38}, SolrDocument{id=39, intGSL=127916419, longGSL=1312574611552010408, doubleGSL=27288.351806523657, floatGSL=15048.335, dateGSL=Tue Sep 17 08:02:28 AKDT 107784622, stringGSL=base_string_447973__00041606, boolGSL=false, sortableGSL=base_string_772653__00030178, _version_=1674636562951831559, _root_=39}, SolrDocument{id=40, intGSL=127916419, longGSL=1312574611552010408, doubleGSL=27288.351806523657, floatGSL=15048.335, dateGSL=Tue Sep 17 08:02:28 AKDT 107784622, stringGSL=base_string_447973__00041606, boolGSL=true, sortableGSL=base_string_772653__00030178, _version_=1674636562952880129, _root_=40}, SolrDocument{id=41, intGSL=127916419, longGSL=1312574611552010408, doubleGSL=27288.351806523657, floatGSL=15048.335, dateGSL=Tue Sep 17 08:02:28 AKDT 107784622, stringGSL=base_string_447973__00041606, boolGSL=false, sortableGSL=base_string_772653__00030178, _version_=1674636562953928710, _root_=41}]}}, {groupValue=127916917, doclist={numFound=6,numFoundExact=true,start=0,maxScore=1.0,docs=[SolrDocument{id=42, intGSL=127916917, longGSL=1312574611552012902, doubleGSL=31956.351806523657, floatGSL=24976.334, dateGSL=Tue Sep 17 08:02:38 AKDT 107784622, stringGSL=base_string_447973__00049523, boolGSL=true, sortableGSL=base_string_772653__00033469, _version_=1674636562952880130, _root_=42}, SolrDocument{id=43, intGSL=127916917, longGSL=1312574611552012902, doubleGSL=31956.351806523657, floatGSL=24976.334, dateGSL=Tue Sep 17 08:02:38 AKDT 107784622, stringGSL=base_string_447973__00049523, boolGSL=false, sortableGSL=base_string_772653__00033469, _version_=1674636562953928711, _root_=43}, SolrDocument{id=44, intGSL=127916917, longGSL=1312574611552012902, doubleGSL=31956.351806523657, floatGSL=24976.334, dateGSL=Tue Sep 17 08:02:38 AKDT 107784622, stringGSL=base_string_447973__00049523, boolGSL=true, sortableGSL=base_string_772653__00033469, _version_=1674636562954977280, _root_=44}, SolrDocument{id=46, intGSL=127916917, longGSL=1312574611552012902, doubleGSL=31956.351806523657, floatGSL=24976.334, dateGSL=Tue Sep 17 08:02:38 AKDT 107784622, stringGSL=base_string_447973__00049523, boolGSL=true, sortableGSL=base_string_772653__00033469, _version_=1674636562954977281, _root_=46}, SolrDocument{id=47, intGSL=127916917, longGSL=1312574611552012902, doubleGSL=31956.351806523657, floatGSL=24976.334, dateGSL=Tue Sep 17 08:02:38 AKDT 107784622, stringGSL=base_string_447973__00049523, boolGSL=false, sortableGSL=base_string_772653__00033469, _version_=1674636562954977282, _root_=47}, SolrDocument{id=48, intGSL=127916917, longGSL=1312574611552012902, doubleGSL=31956.351806523657, floatGSL=24976.334, dateGSL=Tue Sep 17 08:02:38 AKDT 107784622, stringGSL=base_string_447973__00049523, boolGSL=true, sortableGSL=base_string_772653__00033469, _version_=1674636562952880131, _root_=48}]}}]}}}

Stack Trace:
java.lang.AssertionError: Unexpected number of elements in the group for 'intGSL' size: '8' GroupValue: 'null' rsp: {responseHeader={zkConnected=true, status=0, QTime=6, params={q=*:*, group.sort=id asc, _stateVer_=dv_coll:4, group.limit=100, rows=100, wt=javabin, version=2, group.field=intGSL, group=true}}, grouped={intGSL={matches=50, groups=[{groupValue=127893651, doclist={numFound=7,numFoundExact=true,start=0,maxScore=1.0,docs=[SolrDocument{id=0, intGSL=127893651, longGSL=1312574611551989823, doubleGSL=4390.351806523658, floatGSL=2273.3347, dateGSL=Tue Sep 17 08:01:51 AKDT 107784622, stringGSL=base_string_447973__00005883, boolGSL=true, sortableGSL=base_string_772653__00009563, _version_=1674636562951831552, _root_=0}, SolrDocument{id=1, intGSL=127893651, longGSL=1312574611551989823, doubleGSL=4390.351806523658, floatGSL=2273.3347, dateGSL=Tue Sep 17 08:01:51 AKDT 107784622, stringGSL=base_string_447973__00005883, boolGSL=false, sortableGSL=base_string_772653__00009563, _version_=1674636562949734400, _root_=1}, SolrDocument{id=2, intGSL=127893651, longGSL=1312574611551989823, doubleGSL=4390.351806523658, floatGSL=2273.3347, dateGSL=Tue Sep 17 08:01:51 AKDT 107784622, stringGSL=base_string_447973__00005883, boolGSL=true, sortableGSL=base_string_772653__00009563, _version_=1674636562948685824, _root_=2}, SolrDocument{id=3, intGSL=127893651, longGSL=1312574611551989823, doubleGSL=4390.351806523658, floatGSL=2273.3347, dateGSL=Tue Sep 17 08:01:51 AKDT 107784622, stringGSL=base_string_447973__00005883, boolGSL=false, sortableGSL=base_string_772653__00009563, _version_=1674636562950782976, _root_=3}, SolrDocument{id=4, intGSL=127893651, longGSL=1312574611551989823, doubleGSL=4390.351806523658, floatGSL=2273.3347, dateGSL=Tue Sep 17 08:01:51 AKDT 107784622, stringGSL=base_string_447973__00005883, boolGSL=true, sortableGSL=base_string_772653__00009563, _version_=1674636562952880128, _root_=4}, SolrDocument{id=5, intGSL=127893651, longGSL=1312574611551989823, doubleGSL=4390.351806523658, floatGSL=2273.3347, dateGSL=Tue Sep 17 08:01:51 AKDT 107784622, stringGSL=base_string_447973__00005883, boolGSL=false, sortableGSL=base_string_772653__00009563, _version_=1674636562950782976, _root_=5}, SolrDocument{id=6, intGSL=127893651, longGSL=1312574611551989823, doubleGSL=4390.351806523658, floatGSL=2273.3347, dateGSL=Tue Sep 17 08:01:51 AKDT 107784622, stringGSL=base_string_447973__00005883, boolGSL=true, sortableGSL=base_string_772653__00009563, _version_=1674636562950782977, _root_=6}]}}, {groupValue=127893821, doclist={numFound=7,numFoundExact=true,start=0,maxScore=1.0,docs=[SolrDocument{id=10, intGSL=127893821, longGSL=1312574611551992306, doubleGSL=9048.351806523657, floatGSL=4402.3345, dateGSL=Tue Sep 17 08:01:58 AKDT 107784622, stringGSL=base_string_447973__00015653, boolGSL=true, sortableGSL=base_string_772653__00010968, _version_=1674636562951831552, _root_=10}, SolrDocument{id=11, intGSL=127893821, longGSL=1312574611551992306, doubleGSL=9048.351806523657, floatGSL=4402.3345, dateGSL=Tue Sep 17 08:01:58 AKDT 107784622, stringGSL=base_string_447973__00015653, boolGSL=false, sortableGSL=base_string_772653__00010968, _version_=1674636562951831553, _root_=11}, SolrDocument{id=12, intGSL=127893821, longGSL=1312574611551992306, doubleGSL=9048.351806523657, floatGSL=4402.3345, dateGSL=Tue Sep 17 08:01:58 AKDT 107784622, stringGSL=base_string_447973__00015653, boolGSL=true, sortableGSL=base_string_772653__00010968, _version_=1674636562952880130, _root_=12}, SolrDocument{id=13, intGSL=127893821, longGSL=1312574611551992306, doubleGSL=9048.351806523657, floatGSL=4402.3345, dateGSL=Tue Sep 17 08:01:58 AKDT 107784622, stringGSL=base_string_447973__00015653, boolGSL=false, sortableGSL=base_string_772653__00010968, _version_=1674636562951831554, _root_=13}, SolrDocument{id=7, intGSL=127893821, longGSL=1312574611551992306, doubleGSL=9048.351806523657, floatGSL=4402.3345, dateGSL=Tue Sep 17 08:01:58 AKDT 107784622, stringGSL=base_string_447973__00015653, boolGSL=false, sortableGSL=base_string_772653__00010968, _version_=1674636562950782978, _root_=7}, SolrDocument{id=8, intGSL=127893821, longGSL=1312574611551992306, doubleGSL=9048.351806523657, floatGSL=4402.3345, dateGSL=Tue Sep 17 08:01:58 AKDT 107784622, stringGSL=base_string_447973__00015653, boolGSL=true, sortableGSL=base_string_772653__00010968, _version_=1674636562950782976, _root_=8}, SolrDocument{id=9, intGSL=127893821, longGSL=1312574611551992306, doubleGSL=9048.351806523657, floatGSL=4402.3345, dateGSL=Tue Sep 17 08:01:58 AKDT 107784622, stringGSL=base_string_447973__00015653, boolGSL=false, sortableGSL=base_string_772653__00010968, _version_=1674636562950782979, _root_=9}]}}, {groupValue=null, doclist={numFound=8,numFoundExact=true,start=0,maxScore=1.0,docs=[SolrDocument{id=10000, _version_=1674636562949734400, _root_=10000}, SolrDocument{id=10005, _version_=1674636562952880129, _root_=10005}, SolrDocument{id=10010, _version_=1674636562951831552, _root_=10010}, SolrDocument{id=10015, _version_=1674636562951831556, _root_=10015}, SolrDocument{id=10020, _version_=1674636562951831557, _root_=10020}, SolrDocument{id=10025, _version_=1674636562951831554, _root_=10025}, SolrDocument{id=10035, _version_=1674636562951831558, _root_=10035}, SolrDocument{id=10045, _version_=1674636562951831560, _root_=10045}]}}, {groupValue=127894535, doclist={numFound=7,numFoundExact=true,start=0,maxScore=1.0,docs=[SolrDocument{id=14, intGSL=127894535, longGSL=1312574611551997559, doubleGSL=11174.351806523657, floatGSL=4725.3345, dateGSL=Tue Sep 17 08:02:07 AKDT 107784622, stringGSL=base_string_447973__00023825, boolGSL=true, sortableGSL=base_string_772653__00020865, _version_=1674636562951831555, _root_=14}, SolrDocument{id=15, intGSL=127894535, longGSL=1312574611551997559, doubleGSL=11174.351806523657, floatGSL=4725.3345, dateGSL=Tue Sep 17 08:02:07 AKDT 107784622, stringGSL=base_string_447973__00023825, boolGSL=false, sortableGSL=base_string_772653__00020865, _version_=1674636562952880131, _root_=15}, SolrDocument{id=16, intGSL=127894535, longGSL=1312574611551997559, doubleGSL=11174.351806523657, floatGSL=4725.3345, dateGSL=Tue Sep 17 08:02:07 AKDT 107784622, stringGSL=base_string_447973__00023825, boolGSL=true, sortableGSL=base_string_772653__00020865, _version_=1674636562953928704, _root_=16}, SolrDocument{id=17, intGSL=127894535, longGSL=1312574611551997559, doubleGSL=11174.351806523657, floatGSL=4725.3345, dateGSL=Tue Sep 17 08:02:07 AKDT 107784622, stringGSL=base_string_447973__00023825, boolGSL=false, sortableGSL=base_string_772653__00020865, _version_=1674636562950782977, _root_=17}, SolrDocument{id=18, intGSL=127894535, longGSL=1312574611551997559, doubleGSL=11174.351806523657, floatGSL=4725.3345, dateGSL=Tue Sep 17 08:02:07 AKDT 107784622, stringGSL=base_string_447973__00023825, boolGSL=true, sortableGSL=base_string_772653__00020865, _version_=1674636562951831553, _root_=18}, SolrDocument{id=19, intGSL=127894535, longGSL=1312574611551997559, doubleGSL=11174.351806523657, floatGSL=4725.3345, dateGSL=Tue Sep 17 08:02:07 AKDT 107784622, stringGSL=base_string_447973__00023825, boolGSL=false, sortableGSL=base_string_772653__00020865, _version_=1674636562950782978, _root_=19}, SolrDocument{id=20, intGSL=127894535, longGSL=1312574611551997559, doubleGSL=11174.351806523657, floatGSL=4725.3345, dateGSL=Tue Sep 17 08:02:07 AKDT 107784622, stringGSL=base_string_447973__00023825, boolGSL=true, sortableGSL=base_string_772653__00020865, _version_=1674636562953928705, _root_=20}]}}, {groupValue=127898924, doclist={numFound=4,numFoundExact=true,start=0,maxScore=1.0,docs=[SolrDocument{id=22, intGSL=127898924, longGSL=1312574611552003071, doubleGSL=16074.351806523657, floatGSL=7109.3345, dateGSL=Tue Sep 17 08:02:14 AKDT 107784622, stringGSL=base_string_447973__00025921, boolGSL=true, sortableGSL=base_string_772653__00020905, _version_=1674636562953928706, _root_=22}, SolrDocument{id=24, intGSL=127898924, longGSL=1312574611552003071, doubleGSL=16074.351806523657, floatGSL=7109.3345, dateGSL=Tue Sep 17 08:02:14 AKDT 107784622, stringGSL=base_string_447973__00025921, boolGSL=true, sortableGSL=base_string_772653__00020905, _version_=1674636562951831558, _root_=24}, SolrDocument{id=26, intGSL=127898924, longGSL=1312574611552003071, doubleGSL=16074.351806523657, floatGSL=7109.3345, dateGSL=Tue Sep 17 08:02:14 AKDT 107784622, stringGSL=base_string_447973__00025921, boolGSL=true, sortableGSL=base_string_772653__00020905, _version_=1674636562951831555, _root_=26}, SolrDocument{id=27, intGSL=127898924, longGSL=1312574611552003071, doubleGSL=16074.351806523657, floatGSL=7109.3345, dateGSL=Tue Sep 17 08:02:14 AKDT 107784622, stringGSL=base_string_447973__00025921, boolGSL=false, sortableGSL=base_string_772653__00020905, _version_=1674636562951831559, _root_=27}]}}, {groupValue=127906471, doclist={numFound=5,numFoundExact=true,start=0,maxScore=1.0,docs=[SolrDocument{id=28, intGSL=127906471, longGSL=1312574611552009642, doubleGSL=19080.351806523657, floatGSL=10407.335, dateGSL=Tue Sep 17 08:02:19 AKDT 107784622, stringGSL=base_string_447973__00035445, boolGSL=true, sortableGSL=base_string_772653__00025749, _version_=1674636562951831560, _root_=28}, SolrDocument{id=30, intGSL=127906471, longGSL=1312574611552009642, doubleGSL=19080.351806523657, floatGSL=10407.335, dateGSL=Tue Sep 17 08:02:19 AKDT 107784622, stringGSL=base_string_447973__00035445, boolGSL=true, sortableGSL=base_string_772653__00025749, _version_=1674636562951831556, _root_=30}, SolrDocument{id=31, intGSL=127906471, longGSL=1312574611552009642, doubleGSL=19080.351806523657, floatGSL=10407.335, dateGSL=Tue Sep 17 08:02:19 AKDT 107784622, stringGSL=base_string_447973__00035445, boolGSL=false, sortableGSL=base_string_772653__00025749, _version_=1674636562951831557, _root_=31}, SolrDocument{id=32, intGSL=127906471, longGSL=1312574611552009642, doubleGSL=19080.351806523657, floatGSL=10407.335, dateGSL=Tue Sep 17 08:02:19 AKDT 107784622, stringGSL=base_string_447973__00035445, boolGSL=true, sortableGSL=base_string_772653__00025749, _version_=1674636562951831561, _root_=32}, SolrDocument{id=34, intGSL=127906471, longGSL=1312574611552009642, doubleGSL=19080.351806523657, floatGSL=10407.335, dateGSL=Tue Sep 17 08:02:19 AKDT 107784622, stringGSL=base_string_447973__00035445, boolGSL=true, sortableGSL=base_string_772653__00025749, _version_=1674636562953928707, _root_=34}]}}, {groupValue=127916419, doclist={numFound=6,numFoundExact=true,start=0,maxScore=1.0,docs=[SolrDocument{id=35, intGSL=127916419, longGSL=1312574611552010408, doubleGSL=27288.351806523657, floatGSL=15048.335, dateGSL=Tue Sep 17 08:02:28 AKDT 107784622, stringGSL=base_string_447973__00041606, boolGSL=false, sortableGSL=base_string_772653__00030178, _version_=1674636562953928708, _root_=35}, SolrDocument{id=36, intGSL=127916419, longGSL=1312574611552010408, doubleGSL=27288.351806523657, floatGSL=15048.335, dateGSL=Tue Sep 17 08:02:28 AKDT 107784622, stringGSL=base_string_447973__00041606, boolGSL=true, sortableGSL=base_string_772653__00030178, _version_=1674636562953928709, _root_=36}, SolrDocument{id=38, intGSL=127916419, longGSL=1312574611552010408, doubleGSL=27288.351806523657, floatGSL=15048.335, dateGSL=Tue Sep 17 08:02:28 AKDT 107784622, stringGSL=base_string_447973__00041606, boolGSL=true, sortableGSL=base_string_772653__00030178, _version_=1674636562952880128, _root_=38}, SolrDocument{id=39, intGSL=127916419, longGSL=1312574611552010408, doubleGSL=27288.351806523657, floatGSL=15048.335, dateGSL=Tue Sep 17 08:02:28 AKDT 107784622, stringGSL=base_string_447973__00041606, boolGSL=false, sortableGSL=base_string_772653__00030178, _version_=1674636562951831559, _root_=39}, SolrDocument{id=40, intGSL=127916419, longGSL=1312574611552010408, doubleGSL=27288.351806523657, floatGSL=15048.335, dateGSL=Tue Sep 17 08:02:28 AKDT 107784622, stringGSL=base_string_447973__00041606, boolGSL=true, sortableGSL=base_string_772653__00030178, _version_=1674636562952880129, _root_=40}, SolrDocument{id=41, intGSL=127916419, longGSL=1312574611552010408, doubleGSL=27288.351806523657, floatGSL=15048.335, dateGSL=Tue Sep 17 08:02:28 AKDT 107784622, stringGSL=base_string_447973__00041606, boolGSL=false, sortableGSL=base_string_772653__00030178, _version_=1674636562953928710, _root_=41}]}}, {groupValue=127916917, doclist={numFound=6,numFoundExact=true,start=0,maxScore=1.0,docs=[SolrDocument{id=42, intGSL=127916917, longGSL=1312574611552012902, doubleGSL=31956.351806523657, floatGSL=24976.334, dateGSL=Tue Sep 17 08:02:38 AKDT 107784622, stringGSL=base_string_447973__00049523, boolGSL=true, sortableGSL=base_string_772653__00033469, _version_=1674636562952880130, _root_=42}, SolrDocument{id=43, intGSL=127916917, longGSL=1312574611552012902, doubleGSL=31956.351806523657, floatGSL=24976.334, dateGSL=Tue Sep 17 08:02:38 AKDT 107784622, stringGSL=base_string_447973__00049523, boolGSL=false, sortableGSL=base_string_772653__00033469, _version_=1674636562953928711, _root_=43}, SolrDocument{id=44, intGSL=127916917, longGSL=1312574611552012902, doubleGSL=31956.351806523657, floatGSL=24976.334, dateGSL=Tue Sep 17 08:02:38 AKDT 107784622, stringGSL=base_string_447973__00049523, boolGSL=true, sortableGSL=base_string_772653__00033469, _version_=1674636562954977280, _root_=44}, SolrDocument{id=46, intGSL=127916917, longGSL=1312574611552012902, doubleGSL=31956.351806523657, floatGSL=24976.334, dateGSL=Tue Sep 17 08:02:38 AKDT 107784622, stringGSL=base_string_447973__00049523, boolGSL=true, sortableGSL=base_string_772653__00033469, _version_=1674636562954977281, _root_=46}, SolrDocument{id=47, intGSL=127916917, longGSL=1312574611552012902, doubleGSL=31956.351806523657, floatGSL=24976.334, dateGSL=Tue Sep 17 08:02:38 AKDT 107784622, stringGSL=base_string_447973__00049523, boolGSL=false, sortableGSL=base_string_772653__00033469, _version_=1674636562954977282, _root_=47}, SolrDocument{id=48, intGSL=127916917, longGSL=1312574611552012902, doubleGSL=31956.351806523657, floatGSL=24976.334, dateGSL=Tue Sep 17 08:02:38 AKDT 107784622, stringGSL=base_string_447973__00049523, boolGSL=true, sortableGSL=base_string_772653__00033469, _version_=1674636562952880131, _root_=48}]}}]}}}
	at __randomizedtesting.SeedInfo.seed([F7F8692069F61423:420B2D130851717D]:0)
	at org.junit.Assert.fail(Assert.java:88)
	at org.apache.solr.cloud.DocValuesNotIndexedTest.doGroupingDvOnly(DocValuesNotIndexedTest.java:403)
	at org.apache.solr.cloud.DocValuesNotIndexedTest.testGroupingDVOnlySortLast(DocValuesNotIndexedTest.java:331)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:834)


FAILED:  org.apache.solr.servlet.TestRequestRateLimiter.testConcurrentQueries

Error Message:
Incoming accepted new request count did not match. Expected 5 incoming 25

Stack Trace:
java.lang.AssertionError: Incoming accepted new request count did not match. Expected 5 incoming 25
	at __randomizedtesting.SeedInfo.seed([F7F8692069F61423:B5638A05E9916102]:0)
	at org.junit.Assert.fail(Assert.java:88)
	at org.junit.Assert.assertTrue(Assert.java:41)
	at org.apache.solr.servlet.TestRequestRateLimiter.testConcurrentQueries(TestRequestRateLimiter.java:74)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:834)




Build Log:
[...truncated 13306 lines...]
   [junit4] Suite: org.apache.solr.cloud.DocValuesNotIndexedTest
   [junit4]   2> 2224695 INFO  (SUITE-DocValuesNotIndexedTest-seed#[F7F8692069F61423]-worker) [     ] o.a.s.SolrTestCase Setting 'solr.default.confdir' system property to test-framework derived value of '/home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/server/solr/configsets/_default/conf'
   [junit4]   2> 2224696 INFO  (SUITE-DocValuesNotIndexedTest-seed#[F7F8692069F61423]-worker) [     ] o.a.s.SolrTestCaseJ4 Created dataDir: /home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.DocValuesNotIndexedTest_F7F8692069F61423-001/data-dir-89-001
   [junit4]   2> 2224696 WARN  (SUITE-DocValuesNotIndexedTest-seed#[F7F8692069F61423]-worker) [     ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=64 numCloses=64
   [junit4]   2> 2224696 INFO  (SUITE-DocValuesNotIndexedTest-seed#[F7F8692069F61423]-worker) [     ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 2224697 INFO  (SUITE-DocValuesNotIndexedTest-seed#[F7F8692069F61423]-worker) [     ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason="", ssl=0.0/0.0, value=0.0/0.0, clientAuth=0.0/0.0)
   [junit4]   2> 2224697 INFO  (SUITE-DocValuesNotIndexedTest-seed#[F7F8692069F61423]-worker) [     ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 2224697 INFO  (SUITE-DocValuesNotIndexedTest-seed#[F7F8692069F61423]-worker) [     ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.DocValuesNotIndexedTest_F7F8692069F61423-001/tempDir-001
   [junit4]   2> 2224698 INFO  (SUITE-DocValuesNotIndexedTest-seed#[F7F8692069F61423]-worker) [     ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 2224698 INFO  (ZkTestServer Run Thread) [     ] o.a.s.c.ZkTestServer client port: 0.0.0.0/0.0.0.0:0
   [junit4]   2> 2224698 INFO  (ZkTestServer Run Thread) [     ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 2224699 WARN  (ZkTestServer Run Thread) [     ] o.a.z.s.ServerCnxnFactory maxCnxns is not configured, using default value 0.
   [junit4]   2> 2224798 INFO  (SUITE-DocValuesNotIndexedTest-seed#[F7F8692069F61423]-worker) [     ] o.a.s.c.ZkTestServer start zk server on port: 40211
   [junit4]   2> 2224798 INFO  (SUITE-DocValuesNotIndexedTest-seed#[F7F8692069F61423]-worker) [     ] o.a.s.c.ZkTestServer waitForServerUp: 127.0.0.1:40211
   [junit4]   2> 2224798 INFO  (SUITE-DocValuesNotIndexedTest-seed#[F7F8692069F61423]-worker) [     ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:40211
   [junit4]   2> 2224798 INFO  (SUITE-DocValuesNotIndexedTest-seed#[F7F8692069F61423]-worker) [     ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 40211
   [junit4]   2> 2224800 INFO  (SUITE-DocValuesNotIndexedTest-seed#[F7F8692069F61423]-worker) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2224802 INFO  (zkConnectionManagerCallback-9113-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2224802 INFO  (SUITE-DocValuesNotIndexedTest-seed#[F7F8692069F61423]-worker) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2224805 INFO  (SUITE-DocValuesNotIndexedTest-seed#[F7F8692069F61423]-worker) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2224806 INFO  (zkConnectionManagerCallback-9115-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2224806 INFO  (SUITE-DocValuesNotIndexedTest-seed#[F7F8692069F61423]-worker) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2224807 INFO  (SUITE-DocValuesNotIndexedTest-seed#[F7F8692069F61423]-worker) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2224808 INFO  (zkConnectionManagerCallback-9117-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2224808 INFO  (SUITE-DocValuesNotIndexedTest-seed#[F7F8692069F61423]-worker) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2224912 INFO  (jetty-launcher-9118-thread-1) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
   [junit4]   2> 2224912 INFO  (jetty-launcher-9118-thread-2) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
   [junit4]   2> 2224912 INFO  (jetty-launcher-9118-thread-1) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 2 ...
   [junit4]   2> 2224912 INFO  (jetty-launcher-9118-thread-2) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 2 ...
   [junit4]   2> 2224912 INFO  (jetty-launcher-9118-thread-1) [     ] o.e.j.s.Server jetty-9.4.27.v20200227; built: 2020-02-27T18:37:21.340Z; git: a304fd9f351f337e7c0e2a7c28878dd536149c6c; jvm 11.0.4+10-LTS
   [junit4]   2> 2224912 INFO  (jetty-launcher-9118-thread-2) [     ] o.e.j.s.Server jetty-9.4.27.v20200227; built: 2020-02-27T18:37:21.340Z; git: a304fd9f351f337e7c0e2a7c28878dd536149c6c; jvm 11.0.4+10-LTS
   [junit4]   2> 2224914 INFO  (jetty-launcher-9118-thread-1) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 2224914 INFO  (jetty-launcher-9118-thread-1) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 2224914 INFO  (jetty-launcher-9118-thread-1) [     ] o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 2224914 INFO  (jetty-launcher-9118-thread-2) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 2224914 INFO  (jetty-launcher-9118-thread-2) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 2224914 INFO  (jetty-launcher-9118-thread-2) [     ] o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 2224914 INFO  (jetty-launcher-9118-thread-1) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@653addff{/solr,null,AVAILABLE}
   [junit4]   2> 2224915 INFO  (jetty-launcher-9118-thread-2) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@76c74a31{/solr,null,AVAILABLE}
   [junit4]   2> 2224915 INFO  (jetty-launcher-9118-thread-1) [     ] o.e.j.s.AbstractConnector Started ServerConnector@79bb2ec6{HTTP/1.1, (http/1.1, h2c)}{127.0.0.1:46527}
   [junit4]   2> 2224915 INFO  (jetty-launcher-9118-thread-1) [     ] o.e.j.s.Server Started @2224972ms
   [junit4]   2> 2224915 INFO  (jetty-launcher-9118-thread-1) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=46527}
   [junit4]   2> 2224915 INFO  (jetty-launcher-9118-thread-2) [     ] o.e.j.s.AbstractConnector Started ServerConnector@26bb3810{HTTP/1.1, (http/1.1, h2c)}{127.0.0.1:40765}
   [junit4]   2> 2224915 ERROR (jetty-launcher-9118-thread-1) [     ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2224915 INFO  (jetty-launcher-9118-thread-2) [     ] o.e.j.s.Server Started @2224972ms
   [junit4]   2> 2224915 INFO  (jetty-launcher-9118-thread-1) [     ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 2224915 INFO  (jetty-launcher-9118-thread-1) [     ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 9.0.0
   [junit4]   2> 2224915 INFO  (jetty-launcher-9118-thread-2) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=40765}
   [junit4]   2> 2224915 INFO  (jetty-launcher-9118-thread-1) [     ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2224915 INFO  (jetty-launcher-9118-thread-1) [     ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: /home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr
   [junit4]   2> 2224915 INFO  (jetty-launcher-9118-thread-1) [     ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2020-08-10T11:11:17.977856Z
   [junit4]   2> 2224915 ERROR (jetty-launcher-9118-thread-2) [     ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2224915 INFO  (jetty-launcher-9118-thread-2) [     ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 2224915 INFO  (jetty-launcher-9118-thread-2) [     ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 9.0.0
   [junit4]   2> 2224916 INFO  (jetty-launcher-9118-thread-2) [     ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2224916 INFO  (jetty-launcher-9118-thread-2) [     ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: /home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr
   [junit4]   2> 2224916 INFO  (jetty-launcher-9118-thread-2) [     ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2020-08-10T11:11:17.978093Z
   [junit4]   2> 2224916 INFO  (jetty-launcher-9118-thread-1) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2224917 INFO  (jetty-launcher-9118-thread-2) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2224917 INFO  (zkConnectionManagerCallback-9120-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2224918 INFO  (jetty-launcher-9118-thread-1) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2224918 INFO  (zkConnectionManagerCallback-9122-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2224918 INFO  (jetty-launcher-9118-thread-2) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2224918 INFO  (jetty-launcher-9118-thread-1) [     ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 2224918 INFO  (jetty-launcher-9118-thread-2) [     ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 2225026 INFO  (jetty-launcher-9118-thread-2) [     ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
   [junit4]   2> 2225026 WARN  (jetty-launcher-9118-thread-2) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@5e07b092[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2225027 WARN  (jetty-launcher-9118-thread-2) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@5e07b092[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2225029 WARN  (jetty-launcher-9118-thread-2) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@460fdb9e[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2225029 WARN  (jetty-launcher-9118-thread-2) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@460fdb9e[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2225031 INFO  (jetty-launcher-9118-thread-2) [     ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:40211/solr
   [junit4]   2> 2225032 INFO  (jetty-launcher-9118-thread-2) [n:127.0.0.1:40765_solr     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2225033 INFO  (zkConnectionManagerCallback-9132-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2225033 INFO  (jetty-launcher-9118-thread-2) [n:127.0.0.1:40765_solr     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2225126 INFO  (jetty-launcher-9118-thread-1) [     ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
   [junit4]   2> 2225126 WARN  (jetty-launcher-9118-thread-1) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@14995c0[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2225126 WARN  (jetty-launcher-9118-thread-1) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@14995c0[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2225129 WARN  (jetty-launcher-9118-thread-1) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@678404c3[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2225129 WARN  (jetty-launcher-9118-thread-1) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@678404c3[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 2225130 INFO  (jetty-launcher-9118-thread-1) [     ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:40211/solr
   [junit4]   2> 2225131 INFO  (jetty-launcher-9118-thread-1) [n:127.0.0.1:46527_solr     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2225132 INFO  (zkConnectionManagerCallback-9144-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2225132 INFO  (jetty-launcher-9118-thread-1) [n:127.0.0.1:46527_solr     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2225135 INFO  (jetty-launcher-9118-thread-2) [n:127.0.0.1:40765_solr     ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:40765_solr
   [junit4]   2> 2225136 INFO  (jetty-launcher-9118-thread-2) [n:127.0.0.1:40765_solr     ] o.a.s.c.Overseer Overseer (id=72075049243377669-127.0.0.1:40765_solr-n_0000000000) starting
   [junit4]   2> 2225141 INFO  (jetty-launcher-9118-thread-1) [n:127.0.0.1:46527_solr     ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:46527_solr
   [junit4]   2> 2225142 INFO  (OverseerStateUpdate-72075049243377669-127.0.0.1:40765_solr-n_0000000000) [n:127.0.0.1:40765_solr     ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:40765_solr
   [junit4]   2> 2225142 INFO  (jetty-launcher-9118-thread-2) [n:127.0.0.1:40765_solr     ] o.a.s.c.ZkController Publish node=127.0.0.1:40765_solr as DOWN
   [junit4]   2> 2225142 INFO  (zkCallback-9143-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2225143 INFO  (zkCallback-9131-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2225144 INFO  (jetty-launcher-9118-thread-2) [n:127.0.0.1:40765_solr     ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 1000 transient cores
   [junit4]   2> 2225144 INFO  (jetty-launcher-9118-thread-2) [n:127.0.0.1:40765_solr     ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:40765_solr
   [junit4]   2> 2225144 INFO  (jetty-launcher-9118-thread-1) [n:127.0.0.1:46527_solr     ] o.a.s.p.PackageLoader /packages.json updated to version -1
   [junit4]   2> 2225144 WARN  (jetty-launcher-9118-thread-1) [n:127.0.0.1:46527_solr     ] o.a.s.c.CoreContainer Not all security plugins configured!  authentication=disabled authorization=disabled.  Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external.  See https://s.apache.org/solrsecurity for more info
   [junit4]   2> 2225145 INFO  (zkCallback-9143-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2225159 INFO  (zkCallback-9131-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2225160 INFO  (jetty-launcher-9118-thread-1) [n:127.0.0.1:46527_solr     ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 2225161 INFO  (jetty-launcher-9118-thread-2) [n:127.0.0.1:40765_solr     ] o.a.s.p.PackageLoader /packages.json updated to version -1
   [junit4]   2> 2225161 WARN  (jetty-launcher-9118-thread-2) [n:127.0.0.1:40765_solr     ] o.a.s.c.CoreContainer Not all security plugins configured!  authentication=disabled authorization=disabled.  Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external.  See https://s.apache.org/solrsecurity for more info
   [junit4]   2> 2225185 INFO  (jetty-launcher-9118-thread-2) [n:127.0.0.1:40765_solr     ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 2225190 INFO  (jetty-launcher-9118-thread-1) [n:127.0.0.1:46527_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_46527.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@68369197
   [junit4]   2> 2225213 INFO  (jetty-launcher-9118-thread-2) [n:127.0.0.1:40765_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_40765.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@68369197
   [junit4]   2> 2225213 INFO  (jetty-launcher-9118-thread-1) [n:127.0.0.1:46527_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_46527.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@68369197
   [junit4]   2> 2225213 INFO  (jetty-launcher-9118-thread-1) [n:127.0.0.1:46527_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_46527.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@68369197
   [junit4]   2> 2225215 INFO  (jetty-launcher-9118-thread-1) [n:127.0.0.1:46527_solr     ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.DocValuesNotIndexedTest_F7F8692069F61423-001/tempDir-001/node1
   [junit4]   2> 2225226 INFO  (jetty-launcher-9118-thread-2) [n:127.0.0.1:40765_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_40765.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@68369197
   [junit4]   2> 2225226 INFO  (jetty-launcher-9118-thread-2) [n:127.0.0.1:40765_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_40765.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@68369197
   [junit4]   2> 2225227 INFO  (jetty-launcher-9118-thread-2) [n:127.0.0.1:40765_solr     ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.DocValuesNotIndexedTest_F7F8692069F61423-001/tempDir-001/node2
   [junit4]   2> 2225239 INFO  (SUITE-DocValuesNotIndexedTest-seed#[F7F8692069F61423]-worker) [     ] o.a.s.c.MiniSolrCloudCluster waitForAllNodes: numServers=2
   [junit4]   2> 2225239 INFO  (SUITE-DocValuesNotIndexedTest-seed#[F7F8692069F61423]-worker) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 2225241 INFO  (zkConnectionManagerCallback-9165-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 2225241 INFO  (SUITE-DocValuesNotIndexedTest-seed#[F7F8692069F61423]-worker) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 2225242 INFO  (SUITE-DocValuesNotIndexedTest-seed#[F7F8692069F61423]-worker) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2225243 INFO  (SUITE-DocValuesNotIndexedTest-seed#[F7F8692069F61423]-worker) [     ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:40211/solr ready
   [junit4]   2> 2225249 INFO  (qtp2004667993-36040) [n:127.0.0.1:46527_solr     ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=dv_coll&nrtReplicas=1&action=CREATE&numShards=4&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2225253 INFO  (OverseerThreadFactory-9150-thread-1-processing-n:127.0.0.1:40765_solr) [n:127.0.0.1:40765_solr     ] o.a.s.c.a.c.CreateCollectionCmd Create collection dv_coll
   [junit4]   2> 2225362 INFO  (OverseerStateUpdate-72075049243377669-127.0.0.1:40765_solr-n_0000000000) [n:127.0.0.1:40765_solr     ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"dv_coll",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"dv_coll_shard1_replica_n1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:46527/solr",
   [junit4]   2>   "node_name":"127.0.0.1:46527_solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 2225365 INFO  (OverseerStateUpdate-72075049243377669-127.0.0.1:40765_solr-n_0000000000) [n:127.0.0.1:40765_solr     ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"dv_coll",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"dv_coll_shard2_replica_n2",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:40765/solr",
   [junit4]   2>   "node_name":"127.0.0.1:40765_solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 2225368 INFO  (OverseerStateUpdate-72075049243377669-127.0.0.1:40765_solr-n_0000000000) [n:127.0.0.1:40765_solr     ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"dv_coll",
   [junit4]   2>   "shard":"shard3",
   [junit4]   2>   "core":"dv_coll_shard3_replica_n4",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:46527/solr",
   [junit4]   2>   "node_name":"127.0.0.1:46527_solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 2225371 INFO  (OverseerStateUpdate-72075049243377669-127.0.0.1:40765_solr-n_0000000000) [n:127.0.0.1:40765_solr     ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"dv_coll",
   [junit4]   2>   "shard":"shard4",
   [junit4]   2>   "core":"dv_coll_shard4_replica_n6",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:40765/solr",
   [junit4]   2>   "node_name":"127.0.0.1:40765_solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 2225575 INFO  (qtp2004667993-36038) [n:127.0.0.1:46527_solr    x:dv_coll_shard3_replica_n4 ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node7&collection.configName=conf1&newCollection=true&name=dv_coll_shard3_replica_n4&action=CREATE&numShards=4&collection=dv_coll&shard=shard3&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 2225575 INFO  (qtp2004667993-36042) [n:127.0.0.1:46527_solr    x:dv_coll_shard1_replica_n1 ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf1&newCollection=true&name=dv_coll_shard1_replica_n1&action=CREATE&numShards=4&collection=dv_coll&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 2225576 INFO  (qtp2004667993-36038) [n:127.0.0.1:46527_solr    x:dv_coll_shard3_replica_n4 ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 1000 transient cores
   [junit4]   2> 2225576 INFO  (qtp679269231-36043) [n:127.0.0.1:40765_solr    x:dv_coll_shard2_replica_n2 ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node5&collection.configName=conf1&newCollection=true&name=dv_coll_shard2_replica_n2&action=CREATE&numShards=4&collection=dv_coll&shard=shard2&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 2225576 INFO  (qtp679269231-36039) [n:127.0.0.1:40765_solr    x:dv_coll_shard4_replica_n6 ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node8&collection.configName=conf1&newCollection=true&name=dv_coll_shard4_replica_n6&action=CREATE&numShards=4&collection=dv_coll&shard=shard4&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 2225593 INFO  (qtp2004667993-36042) [n:127.0.0.1:46527_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1 ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.0.0
   [junit4]   2> 2225593 INFO  (qtp2004667993-36038) [n:127.0.0.1:46527_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4 ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.0.0
   [junit4]   2> 2225593 INFO  (qtp679269231-36043) [n:127.0.0.1:40765_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2 ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.0.0
   [junit4]   2> 2225594 INFO  (qtp679269231-36039) [n:127.0.0.1:40765_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6 ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.0.0
   [junit4]   2> 2225600 INFO  (qtp679269231-36043) [n:127.0.0.1:40765_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2 ] o.a.s.c.CoreContainer Creating SolrCore 'dv_coll_shard2_replica_n2' using configuration from configset conf1, trusted=true
   [junit4]   2> 2225600 INFO  (qtp2004667993-36042) [n:127.0.0.1:46527_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1 ] o.a.s.c.CoreContainer Creating SolrCore 'dv_coll_shard1_replica_n1' using configuration from configset conf1, trusted=true
   [junit4]   2> 2225600 INFO  (qtp2004667993-36038) [n:127.0.0.1:46527_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4 ] o.a.s.c.CoreContainer Creating SolrCore 'dv_coll_shard3_replica_n4' using configuration from configset conf1, trusted=true
   [junit4]   2> 2225601 INFO  (qtp679269231-36039) [n:127.0.0.1:40765_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6 ] o.a.s.c.CoreContainer Creating SolrCore 'dv_coll_shard4_replica_n6' using configuration from configset conf1, trusted=true
   [junit4]   2> 2225604 INFO  (qtp679269231-36043) [n:127.0.0.1:40765_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2 ] o.a.s.s.IndexSchema Schema name=minimal
   [junit4]   2> 2225604 INFO  (qtp2004667993-36042) [n:127.0.0.1:46527_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1 ] o.a.s.s.IndexSchema Schema name=minimal
   [junit4]   2> 2225604 INFO  (qtp679269231-36039) [n:127.0.0.1:40765_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6 ] o.a.s.s.IndexSchema Schema name=minimal
   [junit4]   2> 2225604 INFO  (qtp2004667993-36038) [n:127.0.0.1:46527_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4 ] o.a.s.s.IndexSchema Schema name=minimal
   [junit4]   2> 2225608 INFO  (qtp2004667993-36042) [n:127.0.0.1:46527_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1 ] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 2225608 INFO  (qtp2004667993-36038) [n:127.0.0.1:46527_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4 ] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 2225608 INFO  (qtp679269231-36043) [n:127.0.0.1:40765_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2 ] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 2225608 INFO  (qtp679269231-36039) [n:127.0.0.1:40765_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6 ] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 2225608 INFO  (qtp2004667993-36038) [n:127.0.0.1:46527_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4 ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_46527.solr.core.dv_coll.shard3.replica_n4' (registry 'solr.core.dv_coll.shard3.replica_n4') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@68369197
   [junit4]   2> 2225608 INFO  (qtp679269231-36039) [n:127.0.0.1:40765_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6 ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_40765.solr.core.dv_coll.shard4.replica_n6' (registry 'solr.core.dv_coll.shard4.replica_n6') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@68369197
   [junit4]   2> 2225608 INFO  (qtp679269231-36043) [n:127.0.0.1:40765_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2 ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_40765.solr.core.dv_coll.shard2.replica_n2' (registry 'solr.core.dv_coll.shard2.replica_n2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@68369197
   [junit4]   2> 2225608 INFO  (qtp2004667993-36042) [n:127.0.0.1:46527_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1 ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_46527.solr.core.dv_coll.shard1.replica_n1' (registry 'solr.core.dv_coll.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@68369197
   [junit4]   2> 2225608 INFO  (qtp2004667993-36038) [n:127.0.0.1:46527_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4 ] o.a.s.c.SolrCore [[dv_coll_shard3_replica_n4] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.DocValuesNotIndexedTest_F7F8692069F61423-001/tempDir-001/node1/dv_coll_shard3_replica_n4], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.DocValuesNotIndexedTest_F7F8692069F61423-001/tempDir-001/node1/dv_coll_shard3_replica_n4/data/]
   [junit4]   2> 2225608 INFO  (qtp679269231-36039) [n:127.0.0.1:40765_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6 ] o.a.s.c.SolrCore [[dv_coll_shard4_replica_n6] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.DocValuesNotIndexedTest_F7F8692069F61423-001/tempDir-001/node2/dv_coll_shard4_replica_n6], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.DocValuesNotIndexedTest_F7F8692069F61423-001/tempDir-001/node2/dv_coll_shard4_replica_n6/data/]
   [junit4]   2> 2225608 INFO  (qtp679269231-36043) [n:127.0.0.1:40765_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2 ] o.a.s.c.SolrCore [[dv_coll_shard2_replica_n2] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.DocValuesNotIndexedTest_F7F8692069F61423-001/tempDir-001/node2/dv_coll_shard2_replica_n2], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.DocValuesNotIndexedTest_F7F8692069F61423-001/tempDir-001/node2/dv_coll_shard2_replica_n2/data/]
   [junit4]   2> 2225608 INFO  (qtp2004667993-36042) [n:127.0.0.1:46527_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1 ] o.a.s.c.SolrCore [[dv_coll_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.DocValuesNotIndexedTest_F7F8692069F61423-001/tempDir-001/node1/dv_coll_shard1_replica_n1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.DocValuesNotIndexedTest_F7F8692069F61423-001/tempDir-001/node1/dv_coll_shard1_replica_n1/data/]
   [junit4]   2> 2225686 INFO  (qtp679269231-36039) [n:127.0.0.1:40765_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6 ] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 2225686 INFO  (qtp679269231-36039) [n:127.0.0.1:40765_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6 ] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 2225686 INFO  (qtp2004667993-36038) [n:127.0.0.1:46527_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4 ] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 2225686 INFO  (qtp2004667993-36038) [n:127.0.0.1:46527_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4 ] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 2225686 INFO  (qtp2004667993-36042) [n:127.0.0.1:46527_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1 ] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 2225686 INFO  (qtp2004667993-36042) [n:127.0.0.1:46527_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1 ] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 2225688 INFO  (qtp679269231-36039) [n:127.0.0.1:40765_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6 ] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 2225688 INFO  (qtp679269231-36039) [n:127.0.0.1:40765_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6 ] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 2225688 INFO  (qtp2004667993-36038) [n:127.0.0.1:46527_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4 ] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 2225688 INFO  (qtp2004667993-36038) [n:127.0.0.1:46527_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4 ] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 2225688 INFO  (qtp679269231-36043) [n:127.0.0.1:40765_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2 ] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 2225688 INFO  (qtp679269231-36043) [n:127.0.0.1:40765_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2 ] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 2225688 INFO  (qtp2004667993-36042) [n:127.0.0.1:46527_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1 ] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 2225688 INFO  (qtp2004667993-36042) [n:127.0.0.1:46527_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1 ] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 2225690 INFO  (qtp679269231-36043) [n:127.0.0.1:40765_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2 ] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 2225690 INFO  (qtp679269231-36043) [n:127.0.0.1:40765_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2 ] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 2225693 INFO  (qtp679269231-36039) [n:127.0.0.1:40765_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6 ] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 2225693 INFO  (qtp2004667993-36038) [n:127.0.0.1:46527_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4 ] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 2225693 INFO  (qtp679269231-36039) [n:127.0.0.1:40765_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6 ] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 2225693 INFO  (qtp2004667993-36042) [n:127.0.0.1:46527_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1 ] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 2225693 INFO  (qtp679269231-36039) [n:127.0.0.1:40765_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6 ] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/conf1/managed-schema
   [junit4]   2> 2225693 INFO  (qtp2004667993-36038) [n:127.0.0.1:46527_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4 ] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 2225694 INFO  (qtp2004667993-36038) [n:127.0.0.1:46527_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4 ] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/conf1/managed-schema
   [junit4]   2> 2225694 INFO  (qtp2004667993-36042) [n:127.0.0.1:46527_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1 ] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 2225694 INFO  (qtp2004667993-36042) [n:127.0.0.1:46527_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1 ] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/conf1/managed-schema
   [junit4]   2> 2225694 INFO  (qtp679269231-36039) [n:127.0.0.1:40765_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6 ] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
   [junit4]   2> 2225695 INFO  (qtp2004667993-36038) [n:127.0.0.1:46527_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4 ] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
   [junit4]   2> 2225695 INFO  (qtp679269231-36039) [n:127.0.0.1:40765_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6 ] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 ms
   [junit4]   2> 2225695 INFO  (qtp2004667993-36042) [n:127.0.0.1:46527_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1 ] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
   [junit4]   2> 2225695 INFO  (qtp679269231-36043) [n:127.0.0.1:40765_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2 ] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 2225695 INFO  (qtp679269231-36039) [n:127.0.0.1:40765_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6 ] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1674636562275500032
   [junit4]   2> 2225695 INFO  (qtp2004667993-36038) [n:127.0.0.1:46527_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4 ] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 ms
   [junit4]   2> 2225695 INFO  (qtp2004667993-36038) [n:127.0.0.1:46527_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4 ] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1674636562275500032
   [junit4]   2> 2225695 INFO  (qtp679269231-36043) [n:127.0.0.1:40765_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2 ] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 2225695 INFO  (qtp2004667993-36042) [n:127.0.0.1:46527_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1 ] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 ms
   [junit4]   2> 2225695 INFO  (qtp679269231-36043) [n:127.0.0.1:40765_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2 ] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/conf1/managed-schema
   [junit4]   2> 2225695 INFO  (qtp2004667993-36042) [n:127.0.0.1:46527_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1 ] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1674636562275500032
   [junit4]   2> 2225696 INFO  (qtp679269231-36043) [n:127.0.0.1:40765_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2 ] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
   [junit4]   2> 2225696 INFO  (qtp679269231-36043) [n:127.0.0.1:40765_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2 ] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 ms
   [junit4]   2> 2225697 INFO  (qtp679269231-36043) [n:127.0.0.1:40765_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2 ] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1674636562277597184
   [junit4]   2> 2225699 INFO  (searcherExecutor-9173-thread-1-processing-n:127.0.0.1:40765_solr x:dv_coll_shard4_replica_n6 c:dv_coll s:shard4 r:core_node8) [n:127.0.0.1:40765_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6 ] o.a.s.c.SolrCore [dv_coll_shard4_replica_n6]  Registered new searcher autowarm time: 0 ms
   [junit4]   2> 2225699 INFO  (searcherExecutor-9171-thread-1-processing-n:127.0.0.1:46527_solr x:dv_coll_shard3_replica_n4 c:dv_coll s:shard3 r:core_node7) [n:127.0.0.1:46527_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4 ] o.a.s.c.SolrCore [dv_coll_shard3_replica_n4]  Registered new searcher autowarm time: 0 ms
   [junit4]   2> 2225699 INFO  (searcherExecutor-9169-thread-1-processing-n:127.0.0.1:46527_solr x:dv_coll_shard1_replica_n1 c:dv_coll s:shard1 r:core_node3) [n:127.0.0.1:46527_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1 ] o.a.s.c.SolrCore [dv_coll_shard1_replica_n1]  Registered new searcher autowarm time: 0 ms
   [junit4]   2> 2225700 INFO  (searcherExecutor-9168-thread-1-processing-n:127.0.0.1:40765_solr x:dv_coll_shard2_replica_n2 c:dv_coll s:shard2 r:core_node5) [n:127.0.0.1:40765_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2 ] o.a.s.c.SolrCore [dv_coll_shard2_replica_n2]  Registered new searcher autowarm time: 0 ms
   [junit4]   2> 2225702 INFO  (qtp679269231-36039) [n:127.0.0.1:40765_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6 ] o.a.s.c.ZkShardTerms Successful update of terms at /collections/dv_coll/terms/shard4 to Terms{values={core_node8=0}, version=0}
   [junit4]   2> 2225703 INFO  (qtp679269231-36039) [n:127.0.0.1:40765_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6 ] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/dv_coll/leaders/shard4
   [junit4]   2> 2225705 INFO  (qtp2004667993-36042) [n:127.0.0.1:46527_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1 ] o.a.s.c.ZkShardTerms Successful update of terms at /collections/dv_coll/terms/shard1 to Terms{values={core_node3=0}, version=0}
   [junit4]   2> 2225705 INFO  (qtp2004667993-36042) [n:127.0.0.1:46527_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/dv_coll/leaders/shard1
   [junit4]   2> 2225706 INFO  (qtp679269231-36043) [n:127.0.0.1:40765_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2 ] o.a.s.c.ZkShardTerms Successful update of terms at /collections/dv_coll/terms/shard2 to Terms{values={core_node5=0}, version=0}
   [junit4]   2> 2225706 INFO  (qtp679269231-36043) [n:127.0.0.1:40765_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2 ] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/dv_coll/leaders/shard2
   [junit4]   2> 2225708 INFO  (qtp2004667993-36038) [n:127.0.0.1:46527_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4 ] o.a.s.c.ZkShardTerms Successful update of terms at /collections/dv_coll/terms/shard3 to Terms{values={core_node7=0}, version=0}
   [junit4]   2> 2225708 INFO  (qtp679269231-36039) [n:127.0.0.1:40765_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6 ] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 2225708 INFO  (qtp2004667993-36038) [n:127.0.0.1:46527_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4 ] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/dv_coll/leaders/shard3
   [junit4]   2> 2225708 INFO  (qtp679269231-36039) [n:127.0.0.1:40765_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6 ] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 2225708 INFO  (qtp679269231-36039) [n:127.0.0.1:40765_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6 ] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:40765/solr/dv_coll_shard4_replica_n6/
   [junit4]   2> 2225709 INFO  (qtp679269231-36039) [n:127.0.0.1:40765_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6 ] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 2225709 INFO  (qtp679269231-36039) [n:127.0.0.1:40765_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6 ] o.a.s.c.SyncStrategy http://127.0.0.1:40765/solr/dv_coll_shard4_replica_n6/ has no replicas
   [junit4]   2> 2225709 INFO  (qtp679269231-36039) [n:127.0.0.1:40765_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6 ] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/dv_coll/leaders/shard4/leader after winning as /collections/dv_coll/leader_elect/shard4/election/72075049243377669-core_node8-n_0000000000
   [junit4]   2> 2225710 INFO  (qtp2004667993-36042) [n:127.0.0.1:46527_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 2225710 INFO  (qtp2004667993-36042) [n:127.0.0.1:46527_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 2225710 INFO  (qtp2004667993-36042) [n:127.0.0.1:46527_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1 ] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:46527/solr/dv_coll_shard1_replica_n1/
   [junit4]   2> 2225711 INFO  (qtp2004667993-36042) [n:127.0.0.1:46527_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1 ] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 2225711 INFO  (qtp2004667993-36042) [n:127.0.0.1:46527_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1 ] o.a.s.c.SyncStrategy http://127.0.0.1:46527/solr/dv_coll_shard1_replica_n1/ has no replicas
   [junit4]   2> 2225711 INFO  (qtp2004667993-36042) [n:127.0.0.1:46527_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/dv_coll/leaders/shard1/leader after winning as /collections/dv_coll/leader_elect/shard1/election/72075049243377670-core_node3-n_0000000000
   [junit4]   2> 2225712 INFO  (qtp679269231-36043) [n:127.0.0.1:40765_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2 ] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 2225712 INFO  (qtp679269231-36043) [n:127.0.0.1:40765_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2 ] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 2225712 INFO  (qtp679269231-36043) [n:127.0.0.1:40765_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2 ] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:40765/solr/dv_coll_shard2_replica_n2/
   [junit4]   2> 2225712 INFO  (qtp679269231-36039) [n:127.0.0.1:40765_solr c:dv_coll s:shard4 r:core_node8 x:dv_coll_shard4_replica_n6 ] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:40765/solr/dv_coll_shard4_replica_n6/ shard4
   [junit4]   2> 2225712 INFO  (qtp679269231-36043) [n:127.0.0.1:40765_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2 ] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 2225713 INFO  (qtp679269231-36043) [n:127.0.0.1:40765_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2 ] o.a.s.c.SyncStrategy http://127.0.0.1:40765/solr/dv_coll_shard2_replica_n2/ has no replicas
   [junit4]   2> 2225713 INFO  (qtp679269231-36043) [n:127.0.0.1:40765_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2 ] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/dv_coll/leaders/shard2/leader after winning as /collections/dv_coll/leader_elect/shard2/election/72075049243377669-core_node5-n_0000000000
   [junit4]   2> 2225714 INFO  (qtp2004667993-36038) [n:127.0.0.1:46527_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4 ] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 2225714 INFO  (qtp2004667993-36038) [n:127.0.0.1:46527_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4 ] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 2225714 INFO  (qtp2004667993-36038) [n:127.0.0.1:46527_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4 ] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:46527/solr/dv_coll_shard3_replica_n4/
   [junit4]   2> 2225714 INFO  (qtp2004667993-36042) [n:127.0.0.1:46527_solr c:dv_coll s:shard1 r:core_node3 x:dv_coll_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:46527/solr/dv_coll_shard1_replica_n1/ shard1
   [junit4]   2> 2225714 INFO  (qtp2004667993-36038) [n:127.0.0.1:46527_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4 ] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 2225715 INFO  (qtp2004667993-36038) [n:127.0.0.1:46527_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4 ] o.a.s.c.SyncStrategy http://127.0.0.1:46527/solr/dv_coll_shard3_replica_n4/ has no replicas
   [junit4]   2> 2225715 INFO  (qtp2004667993-36038) [n:127.0.0.1:46527_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4 ] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/dv_coll/leaders/shard3/leader after winning as /collections/dv_coll/leader_elect/shard3/election/72075049243377670-core_node7-n_0000000000
   [junit4]   2> 2225715 INFO  (qtp679269231-36043) [n:127.0.0.1:40765_solr c:dv_coll s:shard2 r:core_node5 x:dv_coll_shard2_replica_n2 ] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:40765/solr/dv_coll_shard2_replica_n2/ shard2
   [junit4]   2> 2225717 INFO  (qtp2004667993-36038) [n:127.0.0.1:46527_solr c:dv_coll s:shard3 r:core_node7 x:dv_coll_shard3_replica_n4 ] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:46527/solr/dv_coll_shard3_replica_n4/ shard3
   [junit4]   2> 2225819 INFO  (zkCallback-9131-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/dv_coll/state.json] for collection [dv_coll] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2225819 INFO  (zkCallback-9143-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/dv_coll/state.json] for collection [dv_coll] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2225819 INFO  (zkCallback-9131-thread-2) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/dv_coll/state.json] for collection [dv_coll] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2225819 INFO  (zkCallback-9143-thread-2) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/dv_coll/state.json] for collection [dv_coll] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2225819 INFO  (zkCallback-9131-thread-3) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/dv_coll/state.json] for collect

[...truncated too long message...]

-thread-1) [     ] o.e.j.s.AbstractConnector Stopped ServerConnector@235a5cbe{ssl, (ssl, alpn, http/1.1, h2)}{127.0.0.1:0}
   [junit4]   2> 7131996 INFO  (jetty-closer-37262-thread-1) [     ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@4906f469{/solr,null,UNAVAILABLE}
   [junit4]   2> 7131997 INFO  (jetty-closer-37262-thread-1) [     ] o.e.j.s.session node0 Stopped scavenging
   [junit4]   2> 7131998 INFO  (SUITE-TestRequestRateLimiter-seed#[F7F8692069F61423]-worker) [     ] o.a.s.c.ZkTestServer Shutting down ZkTestServer.
   [junit4]   2> 7132242 WARN  (ZkTestServer Run Thread) [     ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	3	/solr/collections/c2/terms/shard1
   [junit4]   2> 	3	/solr/collections/c1/terms/shard1
   [junit4]   2> 	2	/solr/aliases.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	13	/solr/collections/c2/state.json
   [junit4]   2> 	13	/solr/collections/c1/state.json
   [junit4]   2> 	2	/solr/clusterprops.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	21	/solr/overseer/queue
   [junit4]   2> 	7	/solr/overseer/collection-queue-work
   [junit4]   2> 	6	/solr/collections
   [junit4]   2> 	4	/solr/live_nodes
   [junit4]   2> 
   [junit4]   2> 7132259 INFO  (SUITE-TestRequestRateLimiter-seed#[F7F8692069F61423]-worker) [     ] o.a.s.c.ZkTestServer waitForServerDown: 127.0.0.1:34259
   [junit4]   2> 7132260 INFO  (SUITE-TestRequestRateLimiter-seed#[F7F8692069F61423]-worker) [     ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:34259
   [junit4]   2> 7132260 INFO  (SUITE-TestRequestRateLimiter-seed#[F7F8692069F61423]-worker) [     ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 34259
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.servlet.TestRequestRateLimiter_F7F8692069F61423-001
   [junit4]   2> Aug 10, 2020 12:33:05 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene86): {}, docValues:{}, maxPointsInLeafNode=1767, maxMBSortInHeap=6.97878960864655, sim=Asserting(RandomSimilarity(queryNorm=false): {}), locale=zh-Hans-HK, timezone=Canada/Yukon
   [junit4]   2> NOTE: Linux 4.15.0-112-generic amd64/Oracle Corporation 11.0.4 (64-bit)/cpus=8,threads=1,free=229207304,total=536870912
   [junit4]   2> NOTE: All tests run in this JVM: [FieldAnalysisRequestHandlerTest, TestRecovery, JWTVerificationkeyResolverTest, SolrXmlInZkTest, TestExactSharedStatsCache, TestExactSharedStatsCacheCloud, TestTolerantSearch, ConfigSetsAPITest, HighlighterMaxOffsetTest, LeaderTragicEventTest, AnalysisErrorHandlingTest, TestInPlaceUpdatesDistrib, CheckHdfsIndexTest, DistributedSuggestComponentTest, DeleteStatusTest, DistributedFacetPivotLongTailTest, ExitableDirectoryReaderTest, TestRawTransformer, SignificantTermsQParserPluginTest, PeerSyncWithLeaderAndIndexFingerprintCachingTest, SuggesterFSTTest, BadComponentTest, TestSafeXMLParsing, SoftAutoCommitTest, ResolveAnalyzerByNameTest, UUIDUpdateProcessorFallbackTest, DocValuesTest, RegexBoostProcessorTest, HdfsThreadLeakTest, SolrJmxReporterTest, CleanupOldIndexTest, HdfsChaosMonkeySafeLeaderTest, InfoHandlerTest, PrimitiveFieldTypeTest, TestSlowCompositeReaderWrapper, BasicDistributedZkTest, TestSimpleTrackingShardHandler, SolrMetricsIntegrationTest, TestPKIAuthenticationPlugin, SpellCheckCollatorTest, TestMinHashQParser, CustomTermsComponentTest, TestDistributedGrouping, TestBlobHandler, TestReplicationHandler, ResourceLoaderTest, TestManagedSchemaThreadSafety, TestCoreContainer, FullSolrCloudDistribCmdsTest, AliasIntegrationTest, ImplicitSnitchTest, TestElisionMultitermQuery, SuggesterWFSTTest, TestInfoStreamLogging, TestCodecSupport, TestJsonRequest, IndexSchemaRuntimeFieldTest, TestPackages, TestSystemIdResolver, TestPayloadScoreQParserPlugin, BitVectorTest, TestCloudRecovery, TestManagedSynonymFilterFactory, TestDistributedStatsComponentCardinality, TestHalfAndHalfDocValues, TestUnInvertedFieldException, TestNumericTerms64, TestDynamicFieldResource, MergeStrategyTest, TestBulkSchemaAPI, DocValuesNotIndexedTest, ConcurrentDeleteAndCreateCollectionTest, PolyFieldTest, TestCloudJSONFacetSKGEquiv, LeaderElectionTest, SolrShardReporterTest, TestSubQueryTransformer, SolrCoreMetricManagerTest, TestPivotHelperCode, CreateRoutedAliasTest, TestStressVersions, TestCloudSchemaless, TestSQLHandlerNonCloud, TestNoOpRegenerator, TestDFRSimilarityFactory, TestJsonFacets, TestFieldTypeCollectionResource, MetricsHistoryWithAuthIntegrationTest, CdcrVersionReplicationTest, NotRequiredUniqueKeyTest, ConvertedLegacyTest, AuditLoggerIntegrationTest, TestManagedResource, BlockCacheTest, TestBinaryResponseWriter, TestCustomStream, TestMiniSolrCloudClusterSSL, TestDistributedMap, DistributedExpandComponentTest, TestComponentsName, SparseHLLTest, RouteFieldTest, TestXmlQParserPlugin, TestMacros, SubstringBytesRefFilterTest, FuzzySearchTest, ClusterStateTest, JWTAuthPluginIntegrationTest, TestSort, TestIndexingPerformance, TestCloudPhrasesIdentificationComponent, TestExceedMaxTermLength, TestConfigSetsAPIZkFailure, SliceStateTest, ClassificationUpdateProcessorFactoryTest, XmlInterpolationTest, JavabinLoaderTest, PeerSyncTest, TestFieldCacheSort, TestSolrCloudSnapshots, TestEmbeddedSolrServerConstructors, TestNumericRangeQuery32, BinaryUpdateRequestHandlerTest, NestedAtomicUpdateTest, SearchHandlerTest, TestLMJelinekMercerSimilarityFactory, TestJoin, V2StandaloneTest, IgnoreCommitOptimizeUpdateProcessorFactoryTest, TestPullReplica, MultiSolrCloudTestCaseTest, TestClusterProperties, AdminHandlersProxyTest, TestLocalFSCloudBackupRestore, TemplateUpdateProcessorTest, ShowFileRequestHandlerTest, CollectionStateZnodeTest, TestImpersonationWithHadoopAuth, ZkDynamicConfigTest, VersionInfoTest, TestCollectionAPIs, BadIndexSchemaTest, TestPerFieldSimilarity, TestDocumentBuilder, TestTrie, PathHierarchyTokenizerFactoryTest, ProtectedTermFilterFactoryTest, TestCharFilters, TestDeprecatedFilters, TestLuceneMatchVersion, TestReversedWildcardFilterFactory, TestWordDelimiterFilterFactory, TestEmbeddedSolrServerSchemaAPI, ChaosMonkeyNothingIsSafeTest, ChaosMonkeySafeLeaderTest, DistribDocExpirationUpdateProcessorTest, HttpPartitionWithTlogReplicasTest, MultiThreadedOCPTest, NodeMutatorTest, OutOfBoxZkACLAndCredentialsProvidersTest, OverriddenZkACLAndCredentialsProvidersTest, OverseerRolesTest, OverseerTest, RecoveryAfterSoftCommitTest, RecoveryZkTest, ReindexCollectionTest, SSLMigrationTest, SaslZkACLProviderTest, ShardRoutingCustomTest, SplitShardTest, SystemCollectionCompatTest, TestAuthenticationFramework, TestCloudConsistency, TestRandomRequestDistribution, TestRequestForwarding, TestSSLRandomization, TestSegmentSorting, TestSkipOverseerOperations, TestSolrCloudWithDelegationTokens, TestStressCloudBlindAtomicUpdates, TestTlogReplayVsRecovery, TestTlogReplica, TriLevelCompositeIdRoutingTest, UnloadDistributedZkTest, ZkControllerTest, ZkFailoverTest, ZkNodePropsTest, ZkShardTermsTest, AsyncCallRequestStatusResponseTest, CollectionReloadTest, CollectionTooManyReplicasTest, CustomCollectionTest, ShardSplitTest, HDFSCollectionsAPITest, HdfsBasicDistributedZk2Test, HdfsNNFailoverTest, HdfsRecoverLeaseTest, HdfsRecoveryZkTest, HdfsTlogReplayBufferedWhileIndexingTest, HdfsWriteToMultipleCollectionsTest, TestClusterStateMutator, ZkCollectionPropsCachingTest, AlternateDirectoryTest, BlobRepositoryCloudTest, ByteBuffersDirectoryFactoryTest, CachingDirectoryFactoryTest, ConfigureRecoveryStrategyTest, HdfsDirectoryFactoryTest, PluginInfoTest, QueryResultKeyTest, SOLR749Test, SolrCoreCheckLockOnStartupTest, SolrCoreTest, TestConfig, TestConfigOverlay, TestConfigSetImmutable, TestInitParams, TestJmxIntegration, TestLazyCores, TestNRTOpen, TestQuerySenderListener, TestShardHandlerFactory, TestSimpleTextCodec, TestSolrDeletionPolicy1, TestSolrDeletionPolicy2, TestSolrIndexConfig, TestSolrXml, JsonLoaderTest, MoreLikeThisHandlerTest, PingRequestHandlerTest, TestConfigReload, TestReplicationHandlerBackup, TestReplicationHandlerDiskOverFlow, TestSolrConfigHandlerConcurrent, TestStressThreadBackup, CoreMergeIndexesAdminHandlerTest, DaemonStreamApiTest, MBeansHandlerTest, SplitHandlerTest, TestConfigsApi, TestCoreAdminApis, ThreadDumpHandlerTest, DistributedFacetPivotSmallAdvancedTest, DistributedFacetPivotSmallTest, DistributedMLTComponentTest, DistributedQueryComponentCustomSortTest, FacetPivotSmallTest, InfixSuggestersTest, MoreLikeThisComponentTest, PhrasesIdentificationComponentTest, QueryElevationComponentTest, StatsComponentTest, TermVectorComponentDistributedTest, TestLegacyField, TestLegacyNumericUtils, TestNumericTokenStream, TestLogWatcher, TestFacetMethods, TestFaceting, TestIntervalFaceting, TestSchemaField, TestSchemaManager, TestSchemalessBufferedUpdates, TestSortableTextField, TestTextField, TestUseDocValuesAsStored, TestUseDocValuesAsStored2, AnalyticsQueryTest, QueryEqualityTest, QueryParsingTest, SortSpecParsingTest, SpatialFilterTest, TestCollapseQParserPlugin, TestExtendedDismaxParser, TestFilteredDocIdSet, TestFiltering, TestPseudoReturnFields, TestQueryUtils, TestRangeQuery, SpatialHeatmapFacetsTest, GraphQueryTest, TestCloudNestedDocsSort, TestNestedDocsSort, CloudMLTQParserTest, BasicAuthOnSingleNodeTest, MultiDestinationAuditLoggerTest, PKIAuthenticationIntegrationTest, TestSolrCloudWithHadoopAuthPlugin, TestZkAclsWithHadoopAuth, CacheHeaderTest, HttpSolrCallGetCoreTest, ResponseHeaderTest, TestRequestRateLimiter]
   [junit4] Completed [788/864 (2!)] on J2 in 7.86s, 2 tests, 1 failure <<< FAILURES!

[...truncated 52126 lines...]

[JENKINS] Lucene » Lucene-Solr-NightlyTests-master - Build # 3 - Still Unstable!

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://ci-builds.apache.org/job/Lucene/job/Lucene-Solr-NightlyTests-master/3/

3 tests failed.
FAILED:  org.apache.solr.cloud.api.collections.TestHdfsCloudBackupRestore.test

Error Message:
Node 127.0.0.1:44669_solr has 4 replicas. Expected num replicas : 2. state:  DocCollection(hdfsbackuprestore_testok_restored/14)={   "pullReplicas":0,   "replicationFactor":1,   "shards":{     "shard2":{       "range":"0-7fffffff",       "state":"active",       "replicas":{         "core_node2":{           "core":"hdfsbackuprestore_testok_restored_shard2_replica_n1",           "leader":"true",           "core_node_name":"core_node2",           "base_url":"https://127.0.0.1:44669/solr",           "node_name":"127.0.0.1:44669_solr",           "state":"active",           "collection":"hdfsbackuprestore_testok_restored",           "shard":"shard2",           "type":"NRT",           "force_set_state":"false"},         "core_node6":{           "core":"hdfsbackuprestore_testok_restored_shard2_replica_t5",           "core_node_name":"core_node6",           "base_url":"https://127.0.0.1:44669/solr",           "node_name":"127.0.0.1:44669_solr",           "state":"active",           "collection":"hdfsbackuprestore_testok_restored",           "shard":"shard2",           "type":"TLOG",           "force_set_state":"false"}},       "stateTimestamp":"1596958113189495879"},     "shard1":{       "range":"80000000-ffffffff",       "state":"active",       "replicas":{         "core_node4":{           "core":"hdfsbackuprestore_testok_restored_shard1_replica_n3",           "leader":"true",           "core_node_name":"core_node4",           "base_url":"https://127.0.0.1:44669/solr",           "node_name":"127.0.0.1:44669_solr",           "state":"active",           "collection":"hdfsbackuprestore_testok_restored",           "shard":"shard1",           "type":"NRT",           "force_set_state":"false"},         "core_node8":{           "core":"hdfsbackuprestore_testok_restored_shard1_replica_t7",           "core_node_name":"core_node8",           "base_url":"https://127.0.0.1:44669/solr",           "node_name":"127.0.0.1:44669_solr",           "state":"active",           "collection":"hdfsbackuprestore_testok_restored",           "shard":"shard1",           "type":"TLOG",           "force_set_state":"false"}},       "stateTimestamp":"1596958113189534066"}},   "router":{     "name":"compositeId",     "field":"shard_s"},   "nrtReplicas":1,   "tlogReplicas":1}

Stack Trace:
java.lang.AssertionError: Node 127.0.0.1:44669_solr has 4 replicas. Expected num replicas : 2. state: 
DocCollection(hdfsbackuprestore_testok_restored/14)={
  "pullReplicas":0,
  "replicationFactor":1,
  "shards":{
    "shard2":{
      "range":"0-7fffffff",
      "state":"active",
      "replicas":{
        "core_node2":{
          "core":"hdfsbackuprestore_testok_restored_shard2_replica_n1",
          "leader":"true",
          "core_node_name":"core_node2",
          "base_url":"https://127.0.0.1:44669/solr",
          "node_name":"127.0.0.1:44669_solr",
          "state":"active",
          "collection":"hdfsbackuprestore_testok_restored",
          "shard":"shard2",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node6":{
          "core":"hdfsbackuprestore_testok_restored_shard2_replica_t5",
          "core_node_name":"core_node6",
          "base_url":"https://127.0.0.1:44669/solr",
          "node_name":"127.0.0.1:44669_solr",
          "state":"active",
          "collection":"hdfsbackuprestore_testok_restored",
          "shard":"shard2",
          "type":"TLOG",
          "force_set_state":"false"}},
      "stateTimestamp":"1596958113189495879"},
    "shard1":{
      "range":"80000000-ffffffff",
      "state":"active",
      "replicas":{
        "core_node4":{
          "core":"hdfsbackuprestore_testok_restored_shard1_replica_n3",
          "leader":"true",
          "core_node_name":"core_node4",
          "base_url":"https://127.0.0.1:44669/solr",
          "node_name":"127.0.0.1:44669_solr",
          "state":"active",
          "collection":"hdfsbackuprestore_testok_restored",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node8":{
          "core":"hdfsbackuprestore_testok_restored_shard1_replica_t7",
          "core_node_name":"core_node8",
          "base_url":"https://127.0.0.1:44669/solr",
          "node_name":"127.0.0.1:44669_solr",
          "state":"active",
          "collection":"hdfsbackuprestore_testok_restored",
          "shard":"shard1",
          "type":"TLOG",
          "force_set_state":"false"}},
      "stateTimestamp":"1596958113189534066"}},
  "router":{
    "name":"compositeId",
    "field":"shard_s"},
  "nrtReplicas":1,
  "tlogReplicas":1}
	at __randomizedtesting.SeedInfo.seed([991D89747B2758D4:1149B6AED5DB352C]:0)
	at org.junit.Assert.fail(Assert.java:88)
	at org.junit.Assert.assertTrue(Assert.java:41)
	at org.apache.solr.cloud.api.collections.AbstractCloudBackupRestoreTestCase.lambda$testBackupAndRestore$1(AbstractCloudBackupRestoreTestCase.java:374)
	at java.base/java.util.HashMap.forEach(HashMap.java:1336)
	at org.apache.solr.cloud.api.collections.AbstractCloudBackupRestoreTestCase.testBackupAndRestore(AbstractCloudBackupRestoreTestCase.java:373)
	at org.apache.solr.cloud.api.collections.AbstractCloudBackupRestoreTestCase.test(AbstractCloudBackupRestoreTestCase.java:155)
	at org.apache.solr.cloud.api.collections.TestHdfsCloudBackupRestore.test(TestHdfsCloudBackupRestore.java:225)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:834)


FAILED:  org.apache.solr.cloud.api.collections.TestLocalFSCloudBackupRestore.test

Error Message:
Node 127.0.0.1:39251_solr has 4 replicas. Expected num replicas : 2. state:  DocCollection(backuprestore_testok_restored/11)={   "pullReplicas":0,   "replicationFactor":1,   "shards":{     "shard2":{       "range":"0-7fffffff",       "state":"active",       "replicas":{         "core_node2":{           "core":"backuprestore_testok_restored_shard2_replica_n1",           "leader":"true",           "core_node_name":"core_node2",           "base_url":"https://127.0.0.1:39251/solr",           "node_name":"127.0.0.1:39251_solr",           "state":"active",           "collection":"backuprestore_testok_restored",           "shard":"shard2",           "type":"NRT",           "force_set_state":"false"},         "core_node6":{           "core":"backuprestore_testok_restored_shard2_replica_t5",           "core_node_name":"core_node6",           "base_url":"https://127.0.0.1:39251/solr",           "node_name":"127.0.0.1:39251_solr",           "state":"active",           "collection":"backuprestore_testok_restored",           "shard":"shard2",           "type":"TLOG",           "force_set_state":"false"}},       "stateTimestamp":"1596958041273022812"},     "shard1":{       "range":"80000000-ffffffff",       "state":"active",       "replicas":{         "core_node4":{           "core":"backuprestore_testok_restored_shard1_replica_n3",           "leader":"true",           "core_node_name":"core_node4",           "base_url":"https://127.0.0.1:39251/solr",           "node_name":"127.0.0.1:39251_solr",           "state":"active",           "collection":"backuprestore_testok_restored",           "shard":"shard1",           "type":"NRT",           "force_set_state":"false"},         "core_node8":{           "core":"backuprestore_testok_restored_shard1_replica_t7",           "core_node_name":"core_node8",           "base_url":"https://127.0.0.1:39251/solr",           "node_name":"127.0.0.1:39251_solr",           "state":"active",           "collection":"backuprestore_testok_restored",           "shard":"shard1",           "type":"TLOG",           "force_set_state":"false"}},       "stateTimestamp":"1596958041273054130"}},   "router":{     "name":"compositeId",     "field":"shard_s"},   "nrtReplicas":1,   "tlogReplicas":1}

Stack Trace:
java.lang.AssertionError: Node 127.0.0.1:39251_solr has 4 replicas. Expected num replicas : 2. state: 
DocCollection(backuprestore_testok_restored/11)={
  "pullReplicas":0,
  "replicationFactor":1,
  "shards":{
    "shard2":{
      "range":"0-7fffffff",
      "state":"active",
      "replicas":{
        "core_node2":{
          "core":"backuprestore_testok_restored_shard2_replica_n1",
          "leader":"true",
          "core_node_name":"core_node2",
          "base_url":"https://127.0.0.1:39251/solr",
          "node_name":"127.0.0.1:39251_solr",
          "state":"active",
          "collection":"backuprestore_testok_restored",
          "shard":"shard2",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node6":{
          "core":"backuprestore_testok_restored_shard2_replica_t5",
          "core_node_name":"core_node6",
          "base_url":"https://127.0.0.1:39251/solr",
          "node_name":"127.0.0.1:39251_solr",
          "state":"active",
          "collection":"backuprestore_testok_restored",
          "shard":"shard2",
          "type":"TLOG",
          "force_set_state":"false"}},
      "stateTimestamp":"1596958041273022812"},
    "shard1":{
      "range":"80000000-ffffffff",
      "state":"active",
      "replicas":{
        "core_node4":{
          "core":"backuprestore_testok_restored_shard1_replica_n3",
          "leader":"true",
          "core_node_name":"core_node4",
          "base_url":"https://127.0.0.1:39251/solr",
          "node_name":"127.0.0.1:39251_solr",
          "state":"active",
          "collection":"backuprestore_testok_restored",
          "shard":"shard1",
          "type":"NRT",
          "force_set_state":"false"},
        "core_node8":{
          "core":"backuprestore_testok_restored_shard1_replica_t7",
          "core_node_name":"core_node8",
          "base_url":"https://127.0.0.1:39251/solr",
          "node_name":"127.0.0.1:39251_solr",
          "state":"active",
          "collection":"backuprestore_testok_restored",
          "shard":"shard1",
          "type":"TLOG",
          "force_set_state":"false"}},
      "stateTimestamp":"1596958041273054130"}},
  "router":{
    "name":"compositeId",
    "field":"shard_s"},
  "nrtReplicas":1,
  "tlogReplicas":1}
	at __randomizedtesting.SeedInfo.seed([991D89747B2758D4:1149B6AED5DB352C]:0)
	at org.junit.Assert.fail(Assert.java:88)
	at org.junit.Assert.assertTrue(Assert.java:41)
	at org.apache.solr.cloud.api.collections.AbstractCloudBackupRestoreTestCase.lambda$testBackupAndRestore$1(AbstractCloudBackupRestoreTestCase.java:374)
	at java.base/java.util.HashMap.forEach(HashMap.java:1336)
	at org.apache.solr.cloud.api.collections.AbstractCloudBackupRestoreTestCase.testBackupAndRestore(AbstractCloudBackupRestoreTestCase.java:373)
	at org.apache.solr.cloud.api.collections.AbstractCloudBackupRestoreTestCase.test(AbstractCloudBackupRestoreTestCase.java:155)
	at org.apache.solr.cloud.api.collections.TestLocalFSCloudBackupRestore.test(TestLocalFSCloudBackupRestore.java:93)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:834)


FAILED:  org.apache.solr.servlet.TestRequestRateLimiter.testConcurrentQueries

Error Message:
Incoming accepted new request count did not match. Expected 5 incoming 25

Stack Trace:
java.lang.AssertionError: Incoming accepted new request count did not match. Expected 5 incoming 25
	at __randomizedtesting.SeedInfo.seed([991D89747B2758D4:DB866A51FB402DF5]:0)
	at org.junit.Assert.fail(Assert.java:88)
	at org.junit.Assert.assertTrue(Assert.java:41)
	at org.apache.solr.servlet.TestRequestRateLimiter.testConcurrentQueries(TestRequestRateLimiter.java:74)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1754)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:942)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:978)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:819)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:470)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:951)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:887)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:898)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:370)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.lambda$forkTimeoutingTask$0(ThreadLeakControl.java:826)
	at java.base/java.lang.Thread.run(Thread.java:834)




Build Log:
[...truncated 12457 lines...]
   [junit4] Suite: org.apache.solr.cloud.api.collections.TestLocalFSCloudBackupRestore
   [junit4]   2> 218533 INFO  (SUITE-TestLocalFSCloudBackupRestore-seed#[991D89747B2758D4]-worker) [     ] o.a.s.SolrTestCase Setting 'solr.default.confdir' system property to test-framework derived value of '/home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/server/solr/configsets/_default/conf'
   [junit4]   2> 218533 INFO  (SUITE-TestLocalFSCloudBackupRestore-seed#[991D89747B2758D4]-worker) [     ] o.a.s.SolrTestCaseJ4 Created dataDir: /home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.api.collections.TestLocalFSCloudBackupRestore_991D89747B2758D4-001/data-dir-27-001
   [junit4]   2> 218533 WARN  (SUITE-TestLocalFSCloudBackupRestore-seed#[991D89747B2758D4]-worker) [     ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=1 numCloses=1
   [junit4]   2> 218533 INFO  (SUITE-TestLocalFSCloudBackupRestore-seed#[991D89747B2758D4]-worker) [     ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 218534 INFO  (SUITE-TestLocalFSCloudBackupRestore-seed#[991D89747B2758D4]-worker) [     ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason="", ssl=0.0/0.0, value=0.0/0.0, clientAuth=0.0/0.0)
   [junit4]   2> 218535 INFO  (SUITE-TestLocalFSCloudBackupRestore-seed#[991D89747B2758D4]-worker) [     ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 218536 INFO  (SUITE-TestLocalFSCloudBackupRestore-seed#[991D89747B2758D4]-worker) [     ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.api.collections.TestLocalFSCloudBackupRestore_991D89747B2758D4-001/tempDir-001
   [junit4]   2> 218536 INFO  (SUITE-TestLocalFSCloudBackupRestore-seed#[991D89747B2758D4]-worker) [     ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 218536 INFO  (ZkTestServer Run Thread) [     ] o.a.s.c.ZkTestServer client port: 0.0.0.0/0.0.0.0:0
   [junit4]   2> 218536 INFO  (ZkTestServer Run Thread) [     ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 218537 WARN  (ZkTestServer Run Thread) [     ] o.a.z.s.ServerCnxnFactory maxCnxns is not configured, using default value 0.
   [junit4]   2> 218636 INFO  (SUITE-TestLocalFSCloudBackupRestore-seed#[991D89747B2758D4]-worker) [     ] o.a.s.c.ZkTestServer start zk server on port: 45793
   [junit4]   2> 218636 INFO  (SUITE-TestLocalFSCloudBackupRestore-seed#[991D89747B2758D4]-worker) [     ] o.a.s.c.ZkTestServer waitForServerUp: 127.0.0.1:45793
   [junit4]   2> 218637 INFO  (SUITE-TestLocalFSCloudBackupRestore-seed#[991D89747B2758D4]-worker) [     ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:45793
   [junit4]   2> 218637 INFO  (SUITE-TestLocalFSCloudBackupRestore-seed#[991D89747B2758D4]-worker) [     ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 45793
   [junit4]   2> 218639 INFO  (SUITE-TestLocalFSCloudBackupRestore-seed#[991D89747B2758D4]-worker) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 218641 INFO  (zkConnectionManagerCallback-2729-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 218641 INFO  (SUITE-TestLocalFSCloudBackupRestore-seed#[991D89747B2758D4]-worker) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 218644 INFO  (SUITE-TestLocalFSCloudBackupRestore-seed#[991D89747B2758D4]-worker) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 218650 INFO  (zkConnectionManagerCallback-2731-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 218650 INFO  (SUITE-TestLocalFSCloudBackupRestore-seed#[991D89747B2758D4]-worker) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 218651 INFO  (SUITE-TestLocalFSCloudBackupRestore-seed#[991D89747B2758D4]-worker) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 218652 INFO  (zkConnectionManagerCallback-2733-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 218653 INFO  (SUITE-TestLocalFSCloudBackupRestore-seed#[991D89747B2758D4]-worker) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 218760 INFO  (jetty-launcher-2734-thread-2) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
   [junit4]   2> 218760 INFO  (jetty-launcher-2734-thread-2) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 2 ...
   [junit4]   2> 218760 INFO  (jetty-launcher-2734-thread-1) [     ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
   [junit4]   2> 218760 INFO  (jetty-launcher-2734-thread-1) [     ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 2 ...
   [junit4]   2> 218760 INFO  (jetty-launcher-2734-thread-2) [     ] o.e.j.s.Server jetty-9.4.27.v20200227; built: 2020-02-27T18:37:21.340Z; git: a304fd9f351f337e7c0e2a7c28878dd536149c6c; jvm 11.0.4+10-LTS
   [junit4]   2> 218760 INFO  (jetty-launcher-2734-thread-1) [     ] o.e.j.s.Server jetty-9.4.27.v20200227; built: 2020-02-27T18:37:21.340Z; git: a304fd9f351f337e7c0e2a7c28878dd536149c6c; jvm 11.0.4+10-LTS
   [junit4]   2> 218762 INFO  (jetty-launcher-2734-thread-2) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 218762 INFO  (jetty-launcher-2734-thread-2) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 218762 INFO  (jetty-launcher-2734-thread-2) [     ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 218762 INFO  (jetty-launcher-2734-thread-1) [     ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 218762 INFO  (jetty-launcher-2734-thread-1) [     ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 218762 INFO  (jetty-launcher-2734-thread-1) [     ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 218763 INFO  (jetty-launcher-2734-thread-2) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4425f0a6{/solr,null,AVAILABLE}
   [junit4]   2> 218765 INFO  (jetty-launcher-2734-thread-2) [     ] o.e.j.s.AbstractConnector Started ServerConnector@5fc697c3{ssl, (ssl, alpn, http/1.1, h2)}{127.0.0.1:٣٩٢٥١}
   [junit4]   2> 218765 INFO  (jetty-launcher-2734-thread-2) [     ] o.e.j.s.Server Started @٢١٨٨٠٣ms
   [junit4]   2> 218765 INFO  (jetty-launcher-2734-thread-2) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=39251}
   [junit4]   2> 218765 ERROR (jetty-launcher-2734-thread-2) [     ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 218766 INFO  (jetty-launcher-2734-thread-2) [     ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 218766 INFO  (jetty-launcher-2734-thread-2) [     ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 9.0.0
   [junit4]   2> 218766 INFO  (jetty-launcher-2734-thread-2) [     ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 218766 INFO  (jetty-launcher-2734-thread-2) [     ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: /home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr
   [junit4]   2> 218766 INFO  (jetty-launcher-2734-thread-2) [     ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2020-08-09T07:27:12.155158Z
   [junit4]   2> 218767 INFO  (jetty-launcher-2734-thread-1) [     ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@736dc557{/solr,null,AVAILABLE}
   [junit4]   2> 218768 INFO  (jetty-launcher-2734-thread-2) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 218769 INFO  (zkConnectionManagerCallback-2736-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 218769 INFO  (jetty-launcher-2734-thread-2) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 218769 INFO  (jetty-launcher-2734-thread-1) [     ] o.e.j.s.AbstractConnector Started ServerConnector@702ea3ab{ssl, (ssl, alpn, http/1.1, h2)}{127.0.0.1:٤٣٤٥٩}
   [junit4]   2> 218769 INFO  (jetty-launcher-2734-thread-1) [     ] o.e.j.s.Server Started @٢١٨٨٠٧ms
   [junit4]   2> 218769 INFO  (jetty-launcher-2734-thread-1) [     ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=43459}
   [junit4]   2> 218769 ERROR (jetty-launcher-2734-thread-1) [     ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 218769 INFO  (jetty-launcher-2734-thread-1) [     ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 218769 INFO  (jetty-launcher-2734-thread-1) [     ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 9.0.0
   [junit4]   2> 218769 INFO  (jetty-launcher-2734-thread-1) [     ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 218770 INFO  (jetty-launcher-2734-thread-1) [     ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: /home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr
   [junit4]   2> 218770 INFO  (jetty-launcher-2734-thread-2) [     ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 218770 INFO  (jetty-launcher-2734-thread-1) [     ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2020-08-09T07:27:12.159035Z
   [junit4]   2> 218770 INFO  (jetty-launcher-2734-thread-1) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 218774 INFO  (zkConnectionManagerCallback-2738-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 218775 INFO  (jetty-launcher-2734-thread-1) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 218775 INFO  (jetty-launcher-2734-thread-1) [     ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 218876 INFO  (jetty-launcher-2734-thread-2) [     ] o.a.s.c.CoreContainer Allowing use of paths: [_ALL_]
   [junit4]   2> 218876 INFO  (jetty-launcher-2734-thread-2) [     ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
   [junit4]   2> 218878 WARN  (jetty-launcher-2734-thread-2) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@68c42c5e[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 218878 WARN  (jetty-launcher-2734-thread-2) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@68c42c5e[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 218882 WARN  (jetty-launcher-2734-thread-2) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@77c1373c[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 218882 WARN  (jetty-launcher-2734-thread-2) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@77c1373c[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 218883 INFO  (jetty-launcher-2734-thread-2) [     ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:45793/solr
   [junit4]   2> 218884 INFO  (jetty-launcher-2734-thread-2) [n:127.0.0.1:39251_solr     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 218886 INFO  (zkConnectionManagerCallback-2748-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 218886 INFO  (jetty-launcher-2734-thread-2) [n:127.0.0.1:39251_solr     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 218976 INFO  (jetty-launcher-2734-thread-1) [     ] o.a.s.c.CoreContainer Allowing use of paths: [_ALL_]
   [junit4]   2> 218976 INFO  (jetty-launcher-2734-thread-1) [     ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=true]
   [junit4]   2> 218978 WARN  (jetty-launcher-2734-thread-1) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@5aa135cf[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 218978 WARN  (jetty-launcher-2734-thread-1) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@5aa135cf[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 218982 WARN  (jetty-launcher-2734-thread-1) [     ] o.e.j.u.s.S.config Trusting all certificates configured for Client@4c062590[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 218982 WARN  (jetty-launcher-2734-thread-1) [     ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@4c062590[provider=null,keyStore=null,trustStore=null]
   [junit4]   2> 218983 INFO  (jetty-launcher-2734-thread-2) [n:127.0.0.1:39251_solr     ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:39251_solr
   [junit4]   2> 218983 INFO  (jetty-launcher-2734-thread-1) [     ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:45793/solr
   [junit4]   2> 218984 INFO  (jetty-launcher-2734-thread-2) [n:127.0.0.1:39251_solr     ] o.a.s.c.Overseer Overseer (id=72068505746735109-127.0.0.1:39251_solr-n_0000000000) starting
   [junit4]   2> 218984 INFO  (jetty-launcher-2734-thread-1) [n:127.0.0.1:43459_solr     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 218985 INFO  (zkConnectionManagerCallback-2760-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 218985 INFO  (jetty-launcher-2734-thread-1) [n:127.0.0.1:43459_solr     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 218991 INFO  (OverseerStateUpdate-72068505746735109-127.0.0.1:39251_solr-n_0000000000) [n:127.0.0.1:39251_solr     ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:39251_solr
   [junit4]   2> 218991 INFO  (jetty-launcher-2734-thread-2) [n:127.0.0.1:39251_solr     ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:39251_solr
   [junit4]   2> 218993 INFO  (zkCallback-2759-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 218993 INFO  (OverseerStateUpdate-72068505746735109-127.0.0.1:39251_solr-n_0000000000) [n:127.0.0.1:39251_solr     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 218994 INFO  (jetty-launcher-2734-thread-2) [n:127.0.0.1:39251_solr     ] o.a.s.p.PackageLoader /packages.json updated to version -1
   [junit4]   2> 218995 WARN  (jetty-launcher-2734-thread-2) [n:127.0.0.1:39251_solr     ] o.a.s.c.CoreContainer Not all security plugins configured!  authentication=disabled authorization=disabled.  Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external.  See https://s.apache.org/solrsecurity for more info
   [junit4]   2> 218995 INFO  (jetty-launcher-2734-thread-2) [n:127.0.0.1:39251_solr     ] o.a.s.c.b.r.BackupRepositoryFactory Added backup repository with configuration params {type = repository,name = local,class = org.apache.solr.core.backup.repository.LocalFileSystemRepository,attributes = {name=local, class=org.apache.solr.core.backup.repository.LocalFileSystemRepository},}
   [junit4]   2> 218995 INFO  (jetty-launcher-2734-thread-2) [n:127.0.0.1:39251_solr     ] o.a.s.c.b.r.BackupRepositoryFactory Added backup repository with configuration params {type = repository,name = poisioned,class = org.apache.solr.cloud.api.collections.TestLocalFSCloudBackupRestore$PoinsionedRepository,attributes = {default=true, name=poisioned, class=org.apache.solr.cloud.api.collections.TestLocalFSCloudBackupRestore$PoinsionedRepository},}
   [junit4]   2> 218995 INFO  (jetty-launcher-2734-thread-2) [n:127.0.0.1:39251_solr     ] o.a.s.c.b.r.BackupRepositoryFactory Default configuration for backup repository is with configuration params {type = repository,name = poisioned,class = org.apache.solr.cloud.api.collections.TestLocalFSCloudBackupRestore$PoinsionedRepository,attributes = {default=true, name=poisioned, class=org.apache.solr.cloud.api.collections.TestLocalFSCloudBackupRestore$PoinsionedRepository},}
   [junit4]   2> 218995 INFO  (jetty-launcher-2734-thread-1) [n:127.0.0.1:43459_solr     ] o.a.s.c.ZkController Publish node=127.0.0.1:43459_solr as DOWN
   [junit4]   2> 218996 INFO  (jetty-launcher-2734-thread-1) [n:127.0.0.1:43459_solr     ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 1000 transient cores
   [junit4]   2> 218996 INFO  (jetty-launcher-2734-thread-1) [n:127.0.0.1:43459_solr     ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43459_solr
   [junit4]   2> 218997 INFO  (zkCallback-2747-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 218997 INFO  (zkCallback-2759-thread-1) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 218999 INFO  (jetty-launcher-2734-thread-1) [n:127.0.0.1:43459_solr     ] o.a.s.p.PackageLoader /packages.json updated to version -1
   [junit4]   2> 218999 WARN  (jetty-launcher-2734-thread-1) [n:127.0.0.1:43459_solr     ] o.a.s.c.CoreContainer Not all security plugins configured!  authentication=disabled authorization=disabled.  Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external.  See https://s.apache.org/solrsecurity for more info
   [junit4]   2> 219000 INFO  (jetty-launcher-2734-thread-1) [n:127.0.0.1:43459_solr     ] o.a.s.c.b.r.BackupRepositoryFactory Added backup repository with configuration params {type = repository,name = local,class = org.apache.solr.core.backup.repository.LocalFileSystemRepository,attributes = {name=local, class=org.apache.solr.core.backup.repository.LocalFileSystemRepository},}
   [junit4]   2> 219000 INFO  (jetty-launcher-2734-thread-1) [n:127.0.0.1:43459_solr     ] o.a.s.c.b.r.BackupRepositoryFactory Added backup repository with configuration params {type = repository,name = poisioned,class = org.apache.solr.cloud.api.collections.TestLocalFSCloudBackupRestore$PoinsionedRepository,attributes = {default=true, name=poisioned, class=org.apache.solr.cloud.api.collections.TestLocalFSCloudBackupRestore$PoinsionedRepository},}
   [junit4]   2> 219000 INFO  (jetty-launcher-2734-thread-1) [n:127.0.0.1:43459_solr     ] o.a.s.c.b.r.BackupRepositoryFactory Default configuration for backup repository is with configuration params {type = repository,name = poisioned,class = org.apache.solr.cloud.api.collections.TestLocalFSCloudBackupRestore$PoinsionedRepository,attributes = {default=true, name=poisioned, class=org.apache.solr.cloud.api.collections.TestLocalFSCloudBackupRestore$PoinsionedRepository},}
   [junit4]   2> 219011 INFO  (jetty-launcher-2734-thread-2) [n:127.0.0.1:39251_solr     ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 219016 INFO  (jetty-launcher-2734-thread-1) [n:127.0.0.1:43459_solr     ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 219040 INFO  (jetty-launcher-2734-thread-2) [n:127.0.0.1:39251_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_39251.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@631fbc0a
   [junit4]   2> 219042 INFO  (jetty-launcher-2734-thread-1) [n:127.0.0.1:43459_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43459.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@631fbc0a
   [junit4]   2> 219056 INFO  (jetty-launcher-2734-thread-2) [n:127.0.0.1:39251_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_39251.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@631fbc0a
   [junit4]   2> 219056 INFO  (jetty-launcher-2734-thread-2) [n:127.0.0.1:39251_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_39251.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@631fbc0a
   [junit4]   2> 219058 INFO  (jetty-launcher-2734-thread-2) [n:127.0.0.1:39251_solr     ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.api.collections.TestLocalFSCloudBackupRestore_991D89747B2758D4-001/tempDir-001/node2
   [junit4]   2> 219058 INFO  (jetty-launcher-2734-thread-1) [n:127.0.0.1:43459_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43459.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@631fbc0a
   [junit4]   2> 219058 INFO  (jetty-launcher-2734-thread-1) [n:127.0.0.1:43459_solr     ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43459.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@631fbc0a
   [junit4]   2> 219060 INFO  (jetty-launcher-2734-thread-1) [n:127.0.0.1:43459_solr     ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.api.collections.TestLocalFSCloudBackupRestore_991D89747B2758D4-001/tempDir-001/node1
   [junit4]   2> 219071 INFO  (SUITE-TestLocalFSCloudBackupRestore-seed#[991D89747B2758D4]-worker) [     ] o.a.s.c.MiniSolrCloudCluster waitForAllNodes: numServers=2
   [junit4]   2> 219072 INFO  (SUITE-TestLocalFSCloudBackupRestore-seed#[991D89747B2758D4]-worker) [     ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 219073 INFO  (zkConnectionManagerCallback-2781-thread-1) [     ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 219074 INFO  (SUITE-TestLocalFSCloudBackupRestore-seed#[991D89747B2758D4]-worker) [     ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 219075 INFO  (SUITE-TestLocalFSCloudBackupRestore-seed#[991D89747B2758D4]-worker) [     ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 219076 INFO  (SUITE-TestLocalFSCloudBackupRestore-seed#[991D89747B2758D4]-worker) [     ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45793/solr ready
   [junit4]   2> 219102 INFO  (TEST-TestLocalFSCloudBackupRestore.testRestoreFailure-seed#[991D89747B2758D4]) [     ] o.a.s.SolrTestCaseJ4 ###Starting testRestoreFailure
   [junit4]   2> 219159 INFO  (qtp1070878746-6951) [n:127.0.0.1:43459_solr     ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params pullReplicas=0&collection.configName=conf1&name=backuprestore_testfailure&nrtReplicas=1&action=CREATE&numShards=2&tlogReplicas=0&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 219162 INFO  (OverseerThreadFactory-2765-thread-1-processing-n:127.0.0.1:39251_solr) [n:127.0.0.1:39251_solr     ] o.a.s.c.a.c.CreateCollectionCmd Create collection backuprestore_testfailure
   [junit4]   2> 219272 INFO  (OverseerStateUpdate-72068505746735109-127.0.0.1:39251_solr-n_0000000000) [n:127.0.0.1:39251_solr     ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"backuprestore_testfailure",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"backuprestore_testfailure_shard1_replica_n1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:39251/solr",
   [junit4]   2>   "node_name":"127.0.0.1:39251_solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 219276 INFO  (OverseerStateUpdate-72068505746735109-127.0.0.1:39251_solr-n_0000000000) [n:127.0.0.1:39251_solr     ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"backuprestore_testfailure",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"backuprestore_testfailure_shard2_replica_n2",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:43459/solr",
   [junit4]   2>   "node_name":"127.0.0.1:43459_solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 219545 INFO  (qtp1070878746-6949) [n:127.0.0.1:43459_solr    x:backuprestore_testfailure_shard2_replica_n2 ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node4&collection.configName=conf1&newCollection=true&name=backuprestore_testfailure_shard2_replica_n2&action=CREATE&numShards=2&collection=backuprestore_testfailure&shard=shard2&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 219547 INFO  (qtp272396687-6952) [n:127.0.0.1:39251_solr    x:backuprestore_testfailure_shard1_replica_n1 ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf1&newCollection=true&name=backuprestore_testfailure_shard1_replica_n1&action=CREATE&numShards=2&collection=backuprestore_testfailure&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 219547 INFO  (qtp272396687-6952) [n:127.0.0.1:39251_solr    x:backuprestore_testfailure_shard1_replica_n1 ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 1000 transient cores
   [junit4]   2> 219554 INFO  (qtp1070878746-6949) [n:127.0.0.1:43459_solr c:backuprestore_testfailure s:shard2 r:core_node4 x:backuprestore_testfailure_shard2_replica_n2 ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.0.0
   [junit4]   2> 219555 INFO  (qtp272396687-6952) [n:127.0.0.1:39251_solr c:backuprestore_testfailure s:shard1 r:core_node3 x:backuprestore_testfailure_shard1_replica_n1 ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 9.0.0
   [junit4]   2> 219562 INFO  (qtp1070878746-6949) [n:127.0.0.1:43459_solr c:backuprestore_testfailure s:shard2 r:core_node4 x:backuprestore_testfailure_shard2_replica_n2 ] o.a.s.c.CoreContainer Creating SolrCore 'backuprestore_testfailure_shard2_replica_n2' using configuration from configset conf1, trusted=true
   [junit4]   2> 219562 INFO  (qtp272396687-6952) [n:127.0.0.1:39251_solr c:backuprestore_testfailure s:shard1 r:core_node3 x:backuprestore_testfailure_shard1_replica_n1 ] o.a.s.c.CoreContainer Creating SolrCore 'backuprestore_testfailure_shard1_replica_n1' using configuration from configset conf1, trusted=true
   [junit4]   2> 219565 INFO  (qtp272396687-6952) [n:127.0.0.1:39251_solr c:backuprestore_testfailure s:shard1 r:core_node3 x:backuprestore_testfailure_shard1_replica_n1 ] o.a.s.s.IndexSchema Schema name=minimal
   [junit4]   2> 219565 INFO  (qtp1070878746-6949) [n:127.0.0.1:43459_solr c:backuprestore_testfailure s:shard2 r:core_node4 x:backuprestore_testfailure_shard2_replica_n2 ] o.a.s.s.IndexSchema Schema name=minimal
   [junit4]   2> 219567 INFO  (qtp272396687-6952) [n:127.0.0.1:39251_solr c:backuprestore_testfailure s:shard1 r:core_node3 x:backuprestore_testfailure_shard1_replica_n1 ] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 219567 INFO  (qtp1070878746-6949) [n:127.0.0.1:43459_solr c:backuprestore_testfailure s:shard2 r:core_node4 x:backuprestore_testfailure_shard2_replica_n2 ] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 219567 INFO  (qtp272396687-6952) [n:127.0.0.1:39251_solr c:backuprestore_testfailure s:shard1 r:core_node3 x:backuprestore_testfailure_shard1_replica_n1 ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_39251.solr.core.backuprestore_testfailure.shard1.replica_n1' (registry 'solr.core.backuprestore_testfailure.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@631fbc0a
   [junit4]   2> 219567 INFO  (qtp1070878746-6949) [n:127.0.0.1:43459_solr c:backuprestore_testfailure s:shard2 r:core_node4 x:backuprestore_testfailure_shard2_replica_n2 ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43459.solr.core.backuprestore_testfailure.shard2.replica_n2' (registry 'solr.core.backuprestore_testfailure.shard2.replica_n2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@631fbc0a
   [junit4]   2> 219567 INFO  (qtp272396687-6952) [n:127.0.0.1:39251_solr c:backuprestore_testfailure s:shard1 r:core_node3 x:backuprestore_testfailure_shard1_replica_n1 ] o.a.s.c.SolrCore [[backuprestore_testfailure_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.api.collections.TestLocalFSCloudBackupRestore_991D89747B2758D4-001/tempDir-001/node2/backuprestore_testfailure_shard1_replica_n1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.api.collections.TestLocalFSCloudBackupRestore_991D89747B2758D4-001/tempDir-001/node2/backuprestore_testfailure_shard1_replica_n1/data/]
   [junit4]   2> 219567 INFO  (qtp1070878746-6949) [n:127.0.0.1:43459_solr c:backuprestore_testfailure s:shard2 r:core_node4 x:backuprestore_testfailure_shard2_replica_n2 ] o.a.s.c.SolrCore [[backuprestore_testfailure_shard2_replica_n2] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.api.collections.TestLocalFSCloudBackupRestore_991D89747B2758D4-001/tempDir-001/node1/backuprestore_testfailure_shard2_replica_n2], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.api.collections.TestLocalFSCloudBackupRestore_991D89747B2758D4-001/tempDir-001/node1/backuprestore_testfailure_shard2_replica_n2/data/]
   [junit4]   2> 219633 INFO  (qtp272396687-6952) [n:127.0.0.1:39251_solr c:backuprestore_testfailure s:shard1 r:core_node3 x:backuprestore_testfailure_shard1_replica_n1 ] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 219633 INFO  (qtp272396687-6952) [n:127.0.0.1:39251_solr c:backuprestore_testfailure s:shard1 r:core_node3 x:backuprestore_testfailure_shard1_replica_n1 ] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 219635 INFO  (qtp272396687-6952) [n:127.0.0.1:39251_solr c:backuprestore_testfailure s:shard1 r:core_node3 x:backuprestore_testfailure_shard1_replica_n1 ] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 219635 INFO  (qtp272396687-6952) [n:127.0.0.1:39251_solr c:backuprestore_testfailure s:shard1 r:core_node3 x:backuprestore_testfailure_shard1_replica_n1 ] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 219635 INFO  (qtp1070878746-6949) [n:127.0.0.1:43459_solr c:backuprestore_testfailure s:shard2 r:core_node4 x:backuprestore_testfailure_shard2_replica_n2 ] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 219635 INFO  (qtp1070878746-6949) [n:127.0.0.1:43459_solr c:backuprestore_testfailure s:shard2 r:core_node4 x:backuprestore_testfailure_shard2_replica_n2 ] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 219637 INFO  (qtp1070878746-6949) [n:127.0.0.1:43459_solr c:backuprestore_testfailure s:shard2 r:core_node4 x:backuprestore_testfailure_shard2_replica_n2 ] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 219637 INFO  (qtp1070878746-6949) [n:127.0.0.1:43459_solr c:backuprestore_testfailure s:shard2 r:core_node4 x:backuprestore_testfailure_shard2_replica_n2 ] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 219638 INFO  (qtp272396687-6952) [n:127.0.0.1:39251_solr c:backuprestore_testfailure s:shard1 r:core_node3 x:backuprestore_testfailure_shard1_replica_n1 ] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 219638 INFO  (qtp272396687-6952) [n:127.0.0.1:39251_solr c:backuprestore_testfailure s:shard1 r:core_node3 x:backuprestore_testfailure_shard1_replica_n1 ] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 219639 INFO  (qtp272396687-6952) [n:127.0.0.1:39251_solr c:backuprestore_testfailure s:shard1 r:core_node3 x:backuprestore_testfailure_shard1_replica_n1 ] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 ms
   [junit4]   2> 219639 INFO  (qtp272396687-6952) [n:127.0.0.1:39251_solr c:backuprestore_testfailure s:shard1 r:core_node3 x:backuprestore_testfailure_shard1_replica_n1 ] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1674531866440368128
   [junit4]   2> 219640 INFO  (qtp1070878746-6949) [n:127.0.0.1:43459_solr c:backuprestore_testfailure s:shard2 r:core_node4 x:backuprestore_testfailure_shard2_replica_n2 ] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 219640 INFO  (qtp1070878746-6949) [n:127.0.0.1:43459_solr c:backuprestore_testfailure s:shard2 r:core_node4 x:backuprestore_testfailure_shard2_replica_n2 ] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 219641 INFO  (qtp1070878746-6949) [n:127.0.0.1:43459_solr c:backuprestore_testfailure s:shard2 r:core_node4 x:backuprestore_testfailure_shard2_replica_n2 ] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 ms
   [junit4]   2> 219641 INFO  (qtp1070878746-6949) [n:127.0.0.1:43459_solr c:backuprestore_testfailure s:shard2 r:core_node4 x:backuprestore_testfailure_shard2_replica_n2 ] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1674531866442465280
   [junit4]   2> 219642 INFO  (searcherExecutor-2785-thread-1-processing-n:127.0.0.1:39251_solr x:backuprestore_testfailure_shard1_replica_n1 c:backuprestore_testfailure s:shard1 r:core_node3) [n:127.0.0.1:39251_solr c:backuprestore_testfailure s:shard1 r:core_node3 x:backuprestore_testfailure_shard1_replica_n1 ] o.a.s.c.SolrCore [backuprestore_testfailure_shard1_replica_n1]  Registered new searcher autowarm time: 0 ms
   [junit4]   2> 219644 INFO  (searcherExecutor-2783-thread-1-processing-n:127.0.0.1:43459_solr x:backuprestore_testfailure_shard2_replica_n2 c:backuprestore_testfailure s:shard2 r:core_node4) [n:127.0.0.1:43459_solr c:backuprestore_testfailure s:shard2 r:core_node4 x:backuprestore_testfailure_shard2_replica_n2 ] o.a.s.c.SolrCore [backuprestore_testfailure_shard2_replica_n2]  Registered new searcher autowarm time: 0 ms
   [junit4]   2> 219646 INFO  (qtp272396687-6952) [n:127.0.0.1:39251_solr c:backuprestore_testfailure s:shard1 r:core_node3 x:backuprestore_testfailure_shard1_replica_n1 ] o.a.s.c.ZkShardTerms Successful update of terms at /collections/backuprestore_testfailure/terms/shard1 to Terms{values={core_node3=0}, version=0}
   [junit4]   2> 219646 INFO  (qtp272396687-6952) [n:127.0.0.1:39251_solr c:backuprestore_testfailure s:shard1 r:core_node3 x:backuprestore_testfailure_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/backuprestore_testfailure/leaders/shard1
   [junit4]   2> 219647 INFO  (qtp1070878746-6949) [n:127.0.0.1:43459_solr c:backuprestore_testfailure s:shard2 r:core_node4 x:backuprestore_testfailure_shard2_replica_n2 ] o.a.s.c.ZkShardTerms Successful update of terms at /collections/backuprestore_testfailure/terms/shard2 to Terms{values={core_node4=0}, version=0}
   [junit4]   2> 219647 INFO  (qtp1070878746-6949) [n:127.0.0.1:43459_solr c:backuprestore_testfailure s:shard2 r:core_node4 x:backuprestore_testfailure_shard2_replica_n2 ] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/backuprestore_testfailure/leaders/shard2
   [junit4]   2> 219651 INFO  (qtp272396687-6952) [n:127.0.0.1:39251_solr c:backuprestore_testfailure s:shard1 r:core_node3 x:backuprestore_testfailure_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 219651 INFO  (qtp272396687-6952) [n:127.0.0.1:39251_solr c:backuprestore_testfailure s:shard1 r:core_node3 x:backuprestore_testfailure_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 219651 INFO  (qtp272396687-6952) [n:127.0.0.1:39251_solr c:backuprestore_testfailure s:shard1 r:core_node3 x:backuprestore_testfailure_shard1_replica_n1 ] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:39251/solr/backuprestore_testfailure_shard1_replica_n1/
   [junit4]   2> 219652 INFO  (qtp272396687-6952) [n:127.0.0.1:39251_solr c:backuprestore_testfailure s:shard1 r:core_node3 x:backuprestore_testfailure_shard1_replica_n1 ] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 219652 INFO  (qtp272396687-6952) [n:127.0.0.1:39251_solr c:backuprestore_testfailure s:shard1 r:core_node3 x:backuprestore_testfailure_shard1_replica_n1 ] o.a.s.c.SyncStrategy https://127.0.0.1:39251/solr/backuprestore_testfailure_shard1_replica_n1/ has no replicas
   [junit4]   2> 219652 INFO  (qtp272396687-6952) [n:127.0.0.1:39251_solr c:backuprestore_testfailure s:shard1 r:core_node3 x:backuprestore_testfailure_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/backuprestore_testfailure/leaders/shard1/leader after winning as /collections/backuprestore_testfailure/leader_elect/shard1/election/72068505746735109-core_node3-n_0000000000
   [junit4]   2> 219653 INFO  (qtp1070878746-6949) [n:127.0.0.1:43459_solr c:backuprestore_testfailure s:shard2 r:core_node4 x:backuprestore_testfailure_shard2_replica_n2 ] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 219653 INFO  (qtp1070878746-6949) [n:127.0.0.1:43459_solr c:backuprestore_testfailure s:shard2 r:core_node4 x:backuprestore_testfailure_shard2_replica_n2 ] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 219653 INFO  (qtp1070878746-6949) [n:127.0.0.1:43459_solr c:backuprestore_testfailure s:shard2 r:core_node4 x:backuprestore_testfailure_shard2_replica_n2 ] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:43459/solr/backuprestore_testfailure_shard2_replica_n2/
   [junit4]   2> 219653 INFO  (qtp1070878746-6949) [n:127.0.0.1:43459_solr c:backuprestore_testfailure s:shard2 r:core_node4 x:backuprestore_testfailure_shard2_replica_n2 ] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 219653 INFO  (qtp1070878746-6949) [n:127.0.0.1:43459_solr c:backuprestore_testfailure s:shard2 r:core_node4 x:backuprestore_testfailure_shard2_replica_n2 ] o.a.s.c.SyncStrategy https://127.0.0.1:43459/solr/backuprestore_testfailure_shard2_replica_n2/ has no replicas
   [junit4]   2> 219653 INFO  (qtp1070878746-6949) [n:127.0.0.1:43459_solr c:backuprestore_testfailure s:shard2 r:core_node4 x:backuprestore_testfailure_shard2_replica_n2 ] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/backuprestore_testfailure/leaders/shard2/leader after winning as /collections/backuprestore_testfailure/leader_elect/shard2/election/72068505746735110-core_node4-n_0000000000
   [junit4]   2> 219654 INFO  (qtp272396687-6952) [n:127.0.0.1:39251_solr c:backuprestore_testfailure s:shard1 r:core_node3 x:backuprestore_testfailure_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:39251/solr/backuprestore_testfailure_shard1_replica_n1/ shard1
   [junit4]   2> 219656 INFO  (qtp1070878746-6949) [n:127.0.0.1:43459_solr c:backuprestore_testfailure s:shard2 r:core_node4 x:backuprestore_testfailure_shard2_replica_n2 ] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:43459/solr/backuprestore_testfailure_shard2_replica_n2/ shard2
   [junit4]   2> 219758 INFO  (zkCallback-2747-thread-2) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/backuprestore_testfailure/state.json] for collection [backuprestore_testfailure] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 219758 INFO  (zkCallback-2759-thread-2) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/backuprestore_testfailure/state.json] for collection [backuprestore_testfailure] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 219759 INFO  (qtp1070878746-6949) [n:127.0.0.1:43459_solr c:backuprestore_testfailure s:shard2 r:core_node4 x:backuprestore_testfailure_shard2_replica_n2 ] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 219759 INFO  (qtp272396687-6952) [n:127.0.0.1:39251_solr c:backuprestore_testfailure s:shard1 r:core_node3 x:backuprestore_testfailure_shard1_replica_n1 ] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 219761 INFO  (qtp1070878746-6949) [n:127.0.0.1:43459_solr     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node4&collection.configName=conf1&newCollection=true&name=backuprestore_testfailure_shard2_replica_n2&action=CREATE&numShards=2&collection=backuprestore_testfailure&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=216
   [junit4]   2> 219761 INFO  (qtp272396687-6952) [n:127.0.0.1:39251_solr     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf1&newCollection=true&name=backuprestore_testfailure_shard1_replica_n1&action=CREATE&numShards=2&collection=backuprestore_testfailure&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=214
   [junit4]   2> 219764 INFO  (qtp1070878746-6951) [n:127.0.0.1:43459_solr     ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
   [junit4]   2> 219863 INFO  (zkCallback-2747-thread-2) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/backuprestore_testfailure/state.json] for collection [backuprestore_testfailure] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 219863 INFO  (zkCallback-2747-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/backuprestore_testfailure/state.json] for collection [backuprestore_testfailure] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 219863 INFO  (zkCallback-2759-thread-2) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/backuprestore_testfailure/state.json] for collection [backuprestore_testfailure] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 219863 INFO  (zkCallback-2759-thread-1) [     ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/backuprestore_testfailure/state.json] for collection [backuprestore_testfailure] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 219863 INFO  (qtp1070878746-6951) [n:127.0.0.1:43459_solr     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={pullReplicas=0&collection.configName=conf1&name=backuprestore_testfailure&nrtReplicas=1&action=CREATE&numShards=2&tlogReplicas=0&wt=javabin&version=2} status=0 QTime=704
   [junit4]   2> 219871 INFO  (qtp1070878746-6953) [n:127.0.0.1:43459_solr c:backuprestore_testfailure s:shard2 r:core_node4 x:backuprestore_testfailure_shard2_replica_n2 ] o.a.s.c.ZkShardTerms Successful update of terms at /collections/backuprestore_testfailure/terms/shard2 to Terms{values={core_node4=1}, version=1}
   [junit4]   2> 219871 INFO  (qtp1070878746-6953) [n:127.0.0.1:43459_solr c:backuprestore_testfailure s:shard2 r:core_node4 x:backuprestore_testfailure_shard2_replica_n2 ] o.a.s.u.p.LogUpdateProcessorFactory [backuprestore_testfailure_shard2_replica_n2]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[2 (1674531866680492032), 3 (1674531866681540608), 5 (1674531866681540609), 6 (1674531866682589184), 7 (1674531866682589185), 9 (1674531866682589186), 17 (1674531866682589187), 18 (1674531866682589188), 19 (1674531866682589189), 21 (1674531866682589190), ... (19 adds)]} 0 5
   [junit4]   2> 219924 INFO  (qtp272396687-6950) [n:127.0.0.1:39251_solr c:backuprestore_testfailure s:shard1 r:core_node3 x:backuprestore_testfailure_shard1_replica_n1 ] o.a.s.c.ZkShardTerms Successful update of terms at /collections/backuprestore_testfailure/terms/shard1 to Terms{values={core_node3=1}, version=1}
   [junit4]   2> 219924 INFO  (qtp272396687-6950) [n:127.0.0.1:39251_solr c:backuprestore_testfailure s:shard1 r:core_node3 x:backuprestore_testfailure_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [backuprestore_testfailure_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[0 (1674531866735017984), 1 (1674531866737115136), 4 (1674531866737115137), 8 (1674531866737115138), 10 (1674531866737115139), 11 (1674531866737115140), 12 (1674531866737115141), 13 (1674531866737115142), 14 (1674531866737115143), 15 (1674531866737115144), ... (24 adds)]} 0 5
   [junit4]   2> 219992 INFO  (searcherExecutor-2783-thread-1-processing-n:127.0.0.1:43459_solr x:backuprestore_testfailure_shard2_replica_n2 c:backuprestore_testfailure s:shard2 r:core_node4) [n:127.0.0.1:43459_solr c:backuprestore_testfailure s:shard2 r:core_node4 x:backuprestore_testfailure_shard2_replica_n2 ] o.a.s.c.SolrCore [backuprestore_testfailure_shard2_replica_n2]  Registered new searcher autowarm time: 0 ms
   [junit4]   2> 219992 INFO  (qtp1070878746-6953) [n:127.0.0.1:43459_solr c:backuprestore_testfailure s:shard2 r:core_node4 x:backuprestore_testfailure_shard2_replica_n2 ] o.a.s.u.p.LogUpdateProcessorFactory [backuprestore_testfailure_shard2_replica_n2]  webapp=/solr path=/update params={update.distrib=TOLEADER&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=https://127.0.0.1:39251/solr/backuprestore_testfailure_shard1_replica_n1/&commit_end_point=leaders&wt=javabin&version=2&expungeDeletes=false}{commit=} 0 8
   [junit4]   2> 220001 INFO  (searcherExecutor-2785-thread-1-processing-n:127.0.0.1:39251_solr x:backuprestore_testfailure_shard1_replica_n1 c:backuprestore_testfailure s:shard1 r:core_node3) [n:127.0.0.1:39251_solr c:backuprestore_testfailure s:shard1 r:core_node3 x:backuprestore_testfailure_shard1_replica_n1 ] o.a.s.c.SolrCore [backuprestore_testfailure_shard1_replica_n1]  Registered new searcher autowarm time: 0 ms
   [junit4]   2> 220001 INFO  (qtp272396687-6948) [n:127.0.0.1:39251_solr c:backuprestore_testfailure s:shard1 r:core_node3 x:backuprestore_testfailure_shard1_replica_n1 ] o.a.s.u.p.LogUpdateProcessorFactory [backuprestore_testfailure_shard1_replica_n1]  webapp=/solr path=/update params={_stateVer_=backuprestore_testfailure:4&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}{commit=} 0 75
   [junit4]   2> 220001 INFO  (TEST-TestLocalFSCloudBackupRestore.testRestoreFailure-seed#[991D89747B2758D4]) [     ] o.a.s.c.a.c.AbstractCloudBackupRestoreTestCase Indexed 43 docs to collection: backuprestore_testfailure
   [junit4]   2> 220002 INFO  (TEST-TestLocalFSCloudBackupRestore.testRestoreFailure-seed#[991D89747B2758D4]) [     ] o.a.s.c.a.c.AbstractCloudBackupRestoreTestCase Triggering Backup command
   [junit4]   2> 220003 INFO  (qtp1070878746-6949) [n:127.0.0.1:43459_solr     ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :backup with params name=mytestbackuptestfailure&action=BACKUP&location=/home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.api.collections.TestLocalFSCloudBackupRestore_991D89747B2758D4-001/my+backup-001&collection=backuprestore_testfailure&repository=local&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 220008 INFO  (OverseerCollectionConfigSetProcessor-72068505746735109-127.0.0.1:39251_solr-n_0000000000) [n:127.0.0.1:39251_solr     ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
   [junit4]   2> 220009 INFO  (OverseerThreadFactory-2765-thread-2-processing-n:127.0.0.1:39251_solr) [n:127.0.0.1:39251_solr c:backuprestore_testfailure    ] o.a.s.c.a.c.BackupCmd Starting backup of collection=backuprestore_testfailure with backupName=mytestbackuptestfailure at location=file:///home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.api.collections.TestLocalFSCloudBackupRestore_991D89747B2758D4-001/my%20backup-001/mytestbackuptestfailure
   [junit4]   2> 220011 INFO  (qtp1070878746-6951) [n:127.0.0.1:43459_solr c:backuprestore_testfailure s:shard2 r:core_node4 x:backuprestore_testfailure_shard2_replica_n2 ] o.a.s.h.SnapShooter Creating backup snapshot shard2 at file:///home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.api.collections.TestLocalFSCloudBackupRestore_991D89747B2758D4-001/my%20backup-001/mytestbackuptestfailure
   [junit4]   2> 220011 INFO  (qtp272396687-6952) [n:127.0.0.1:39251_solr c:backuprestore_testfailure s:shard1 r:core_node3 x:backuprestore_testfailure_shard1_replica_n1 ] o.a.s.h.SnapShooter Creating backup snapshot shard1 at file:///home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.api.collections.TestLocalFSCloudBackupRestore_991D89747B2758D4-001/my%20backup-001/mytestbackuptestfailure
   [junit4]   2> 220018 INFO  (qtp1070878746-6951) [n:127.0.0.1:43459_solr c:backuprestore_testfailure s:shard2 r:core_node4 x:backuprestore_testfailure_shard2_replica_n2 ] o.a.s.h.SnapShooter Done creating backup snapshot: shard2 into file:///home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.api.collections.TestLocalFSCloudBackupRestore_991D89747B2758D4-001/my%20backup-001/mytestbackuptestfailure/snapshot.shard2
   [junit4]   2> 220018 INFO  (qtp1070878746-6951) [n:127.0.0.1:43459_solr     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={core=backuprestore_testfailure_shard2_replica_n2&qt=/admin/cores&name=shard2&action=BACKUPCORE&location=file:///home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.api.collections.TestLocalFSCloudBackupRestore_991D89747B2758D4-001/my%2520backup-001/mytestbackuptestfailure&repository=local&wt=javabin&version=2} status=0 QTime=7
   [junit4]   2> 220018 INFO  (qtp272396687-6952) [n:127.0.0.1:39251_solr c:backuprestore_testfailure s:shard1 r:core_node3 x:backuprestore_testfailure_shard1_replica_n1 ] o.a.s.h.SnapShooter Done creating backup snapshot: shard1 into file:///home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.api.collections.TestLocalFSCloudBackupRestore_991D89747B2758D4-001/my%20backup-001/mytestbackuptestfailure/snapshot.shard1
   [junit4]   2> 220018 INFO  (qtp272396687-6952) [n:127.0.0.1:39251_solr     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={core=backuprestore_testfailure_shard1_replica_n1&qt=/admin/cores&name=shard1&action=BACKUPCORE&location=file:///home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.api.collections.TestLocalFSCloudBackupRestore_991D89747B2758D4-001/my%2520backup-001/mytestbackuptestfailure&repository=local&wt=javabin&version=2} status=0 QTime=7
   [junit4]   2> 220019 INFO  (OverseerThreadFactory-2765-thread-2-processing-n:127.0.0.1:39251_solr) [n:127.0.0.1:39251_solr c:backuprestore_testfailure    ] o.a.s.c.a.c.BackupCmd Starting to backup ZK data for backupName=mytestbackuptestfailure
   [junit4]   2> 220023 INFO  (OverseerThreadFactory-2765-thread-2-processing-n:127.0.0.1:39251_solr) [n:127.0.0.1:39251_solr c:backuprestore_testfailure    ] o.a.s.c.a.c.BackupCmd Completed backing up ZK data for backupName=mytestbackuptestfailure
   [junit4]   2> 220024 INFO  (qtp1070878746-6949) [n:127.0.0.1:43459_solr c:backuprestore_testfailure    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={name=mytestbackuptestfailure&action=BACKUP&location=/home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.api.collections.TestLocalFSCloudBackupRestore_991D89747B2758D4-001/my+backup-001&collection=backuprestore_testfailure&repository=local&wt=javabin&version=2} status=0 QTime=21
   [junit4]   2> 220025 INFO  (TEST-TestLocalFSCloudBackupRestore.testRestoreFailure-seed#[991D89747B2758D4]) [     ] o.a.s.c.a.c.AbstractCloudBackupRestoreTestCase Triggering Restore command
   [junit4]   2> 220027 INFO  (qtp1070878746-6953) [n:127.0.0.1:43459_solr     ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :restore with params async=fd77b6c8-e0aa-4c05-8dbc-b5c3a987226b&collection.configName=confFaulty&name=mytestbackuptestfailure&action=RESTORE&location=/home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.api.collections.TestLocalFSCloudBackupRestore_991D89747B2758D4-001/my+backup-001&collection=backuprestore_testfailure_restored&repository=local&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 220029 INFO  (qtp1070878746-6953) [n:127.0.0.1:43459_solr c:backuprestore_testfailure_restored    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={async=fd77b6c8-e0aa-4c05-8dbc-b5c3a987226b&collection.configName=confFaulty&name=mytestbackuptestfailure&action=RESTORE&location=/home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.api.collections.TestLocalFSCloudBackupRestore_991D89747B2758D4-001/my+backup-001&collection=backuprestore_testfailure_restored&repository=local&wt=javabin&version=2} status=0 QTime=2
   [junit4]   2> 220033 INFO  (qtp1070878746-6951) [n:127.0.0.1:43459_solr     ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :requeststatus with params requestid=fd77b6c8-e0aa-4c05-8dbc-b5c3a987226b&action=REQUESTSTATUS&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 220034 INFO  (OverseerCollectionConfigSetProcessor-72068505746735109-127.0.0.1:39251_solr-n_0000000000) [n:127.0.0.1:39251_solr     ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may have disconnected from ZooKeeper
   [junit4]   2> 220035 INFO  (qtp1070878746-6951) [n:127.0.0.1:43459_solr     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={requestid=fd77b6c8-e0aa-4c05-8dbc-b5c3a987226b&action=REQUESTSTATUS&wt=javabin&version=2} status=0 QTime=1
   [junit4]   2> 220035 INFO  (OverseerThreadFactory-2765-thread-3-processing-n:127.0.0.1:39251_solr) [n:127.0.0.1:39251_solr c:backuprestore_testfailure_restored    ] o.a.s.c.a.c.RestoreCmd Using existing config confFaulty
   [junit4]   2> 220035 INFO  (OverseerThreadFactory-2765-thread-3-processing-n:127.0.0.1:39251_solr) [n:127.0.0.1:39251_solr c:backuprestore_testfailure_restored    ] o.a.s.c.a.c.RestoreCmd Starting restore into collection=backuprestore_testfailure_restored with backup_name=mytestbackuptestfailure at location=file:///home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.api.collections.TestLocalFSCloudBackupRestore_991D89747B2758D4-001/my%20backup-001/
   [junit4]   2> 220036 INFO  (OverseerThreadFactory-2765-thread-3-processing-n:127.0.0.1:39251_solr) [n:127.0.0.1:39251_solr c:backuprestore_testfailure_restored    ] o.a.s.c.a.c.CreateCollectionCmd Create collection backuprestore_testfailure_restored
   [junit4]   2> 220241 WARN  (OverseerThreadFactory-2765-thread-3-processing-n:127.0.0.1:39251_solr) [n:127.0.0.1:39251_solr c:backuprestore_testfailure_restored    ] o.a.s.c.a.c.CreateCollectionCmd It is unusual to create a collection (backuprestore_testfailure_restored) without cores.
   [junit4]   2> 220244 INFO  (OverseerStateUpdate-72068505746735109-127.0.0.1:39251_solr-n_0000000000) [n:127.0.0.1:39251_solr     ] o.a.s.c.o.SliceMutator Update shard state invoked for collection: backuprestore_testfailure_restored with message: {
   [junit4]   2>   "shard2":"construction",
   [junit4]   2>   "shard1":"construction",
   [junit4]   2>   "collection":"backuprestore_testfailure_restored",
   [junit4]   2>   "operation":"updateshardstate"}
   [junit4]   2> 220244 INFO  (OverseerStateUpdate-72068505746735109-127.0.0.1:39251_solr-n_0000000000) [n:127.0.0.1:39251_solr     ] o.a.s.c.o.SliceMutator Update shard state shard2 to construction
   [junit4]   2> 220244 INFO  (OverseerStateUpdate-72068505746735109-127.0.0.1:39251_solr-n_0000000000) [n:127.0.0.1:39251_solr     ] o.a.s.c.o.SliceMutator Update shard state shard1 to construction
   [junit4]   2> 220244 INFO  (OverseerThreadFactory-2765-thread-3-processing-n:127.0.0.1:39251_solr) [n:127.0.0.1:39251_solr c:backuprestore_testfailure_restored    ] o.a.s.c.a.c.RestoreCmd Adding replica for shard=shard2 collection=DocCollection(backuprestore_testfailure_restored/0)={
   [junit4]   2>   "pullReplicas":0,
   [junit4]   2>   "replicationFactor":1,
   [junit4]   2>   "shards":{
   [junit4]   2>     "shard2":{
   [junit4]   2>       "range":"0-7fffffff",
   [junit4]   2>       "state":"active",
   [junit4]   2>       "replicas":{}},
   [junit4]   2>     "shard1":{
   [junit4]   2>       "range":"80000000-ffffffff",
   [junit4]   2>       "state":"active",
   [junit4]   2>       "replicas":{}}},
   [junit4]   2>   "router":{"name":"compositeId"},
   [junit4]   2>   "nrtReplicas":1,
   [junit4]   2>   "tlogReplicas":0} 
   [junit4]   2> 220247 INFO  (OverseerThreadFactory-2765-thread-3-processing-n:127.0.0.1:39251_solr) [n:127.0.0.1:39251_solr c:backuprestore_testfailure_restored    ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:43459_solr for creating new replica of shard shard2 for collection backuprestore_testfailure_restored
   [junit4]   2> 220249 INFO  (OverseerThreadFactory-2765-thread-3-processing-n:127.0.0.1:39251_solr) [n:127.0.0.1:39251_solr c:backuprestore_testfailure_restored    ] o.a.s.c.a.c.AddReplicaCmd Returning CreateReplica command.
   [junit4]   2> 220251 INFO  (OverseerStateUpdate-72068505746735109-127.0.0.1:39251_solr-n_0000000000) [n:127.0.0.1:39251_solr     ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"addreplica",
   [junit4]   2>   "collection":"backuprestore_testfailure_restored",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"backuprestore_testfailure_restored_shard2_replica_n1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:43459/solr",
   [junit4]   2>   "node_name":"127.0.0.1:43459_solr",
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 220458 INFO  (qtp1070878746-6949) [n:127.0.0.1:43459_solr    x:backuprestore_testfailure_restored_shard2_replica_n1 ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={async=fd77b6c8-e0aa-4c05-8dbc-b5c3a987226b166501378538554&qt=/admin/cores&coreNodeName=core_node2&collection.configName=confFaulty&name=backuprestore_testfailure_restored_shard2_replica_n1&action=CREATE&collection=backuprestore_testfailure_restored&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1
   [junit4]   2> 220458 INFO  (parallelCoreAdminExecutor-2768-thread-1-processing-n:127.0.0.1:43459_solr x:backuprestore_testfailure_restored_shard2_replica_n1 fd77b6c8-e0aa-4c05-8dbc-b5c3a987226b166501378538554 CREATE) [n:127.0.0.1:43459_solr    x:backuprestore_testfailure_restored_shard2_replica_n1 ] o.a.s.h.a.CoreAdminOperation core create command async=fd77b6c8-e0aa-4c05-8dbc-b5c3a987226b166501378538554&qt=/admin/cores&coreNodeName=core_node2&collection.configName=confFaulty&name=backuprestore_testfailure_restored_shard2_replica_n1&action=CREATE&collection=backuprestore_testfailure_restored&shard=shard2&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 220459 INFO  (qtp1070878746-6953) [n:127.0.0.1:43459_solr     ] o.a.s.h.a.CoreAdminOperation Checking request status for : fd77b6c8-e0aa-4c05-8dbc-b5c3a987226b166501378538554
   [junit4]   2> 220460 INFO  (qtp1070878746-6953) [n:127.0.0.1:43459_solr     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&requestid=fd77b6c8-e0aa-4c05-8dbc-b5c3a987226b166501378538554&action=REQUESTSTATUS&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 221036 INFO  (qtp1070878746-6951) [n:127.0.0.1:43459_solr     ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :requeststatus with params requestid=fd77b6c8-e0aa-4c05-8dbc-b5c3a987226b&action=REQUESTSTATUS&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 221037 INFO  (qtp1070878746-6951) [n:127.0.0.1:43459_solr     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={requestid=fd77b6c8-e0aa-4c05-8dbc-b5c3a987226b&action=REQUESTSTATUS&wt=javabin&version=2} status=0 QTime=1
   [junit4]   2> 221462 INFO  (qtp1070878746-6949) [n:127.0.0.1:43459_solr     ] o.a.s.h.a.CoreAdminOperation Checking request status for : fd77b6c8-e0aa-4c05-8dbc-b5c3a987226b166501378538554
   [junit4]   2> 221462 INFO  (qtp1070878746-6949) [n:127.0.0.1:43459_solr     ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&requestid=fd77b6c8-e0aa-4c05-8dbc-b5c3a987226b166501378538554&action=REQUESTSTATUS&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 221463 ERROR (OverseerThreadFactory-2765-thread-3-processing-n:127.0.0.1:39251_solr) [n:127.0.0.1:39251_solr c:backuprestore_testfailure_restored    ] o.a.s.c.a.c.OverseerCollectionMessageHandler Error from shard 127.0.0.1:43459_solr: {responseHeader={status=0, QTime=0}, STATUS=failed, Response=Error CREATEing SolrCore 'backuprestore_testfailure_restored_shard2_replica_n1': Unable to create core [backuprestore_testfailure_restored_shard2_replica_n1] Caused by: Can't find resource 'solrconfig.xml' in classpath or '/configs/confFaulty', cwd=/home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2}
   [junit4]   2> 221464 INFO  (OverseerThreadFactory-2765-thread-3-processing-n:127.0.0.1:39251_solr) [n:127.0.0.1:39251_solr c:backuprestore_testfailure_restored    ] o.a.s.c.a.c.RestoreCmd Adding replica for shard=shard1 collection=DocCollection(backuprestore_testfailure_restored/0)={
   [junit4]   2>   "pullReplicas":0,
   [junit4]   2>   "replicationFactor":1,
   [junit4]   2>   "shards":{
   [junit4]   2>     "shard2":{
   [junit4]   2>       "range":"0-7fffffff",
   [junit4]   2>       "state":"active",
   [junit4]   2>       "replicas":{}},
   [junit4]   2>     "shard1":{
   [junit4]   2>       "range":"80000000-ffffffff",
   [junit4]   2>       "state":"active",
   [junit4]   2>       "replicas":{}}},
   [junit4]   2>   "router":{"name":"compositeId"},
   [junit4]   2>   "nrtReplicas":1,
   [junit4]   2>   "tlogReplicas":0} 
   [junit4]   2> 221465 INFO  (OverseerThreadFactory-2765-thread-3-processing-n:127.0.0.1:39251_solr) [n:127.0.0.1:39251_solr c:backuprestore_testfailure_restored    ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:39251_solr for creating new replica of shard shard1 for collection backuprestore_testfailure_restored
   [junit4]   2> 221467 INFO  (OverseerThreadFactory-2765-thread-3-processing-n:127.0.0.1:39251_solr) [n:127.0.0.1:39251_solr c:backuprestore_testfailure_restored    ] o.a.s.c.a.c.AddReplicaCmd Returning CreateReplica command.
   [junit4]   2> 221469 INFO  (OverseerStateUpdate-72068505746735109-127.0.0.1:39251_solr-n_0000000000) [n:127.0.0.1:39251_solr     ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"addreplica",
   [junit4]   2>   "collection":"backuprestore_testfailure_restored",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"backuprestore_testfailure_restored_shard1_replica_n3",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:39251/solr",
   [junit4]   2>   "node_name":"127.0.0.1:39251_solr",
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 221672 INFO  (qtp272396687-6950) [n:127.0.0.1:39251_solr    x:backuprestore_testfailure_restored_shard1_replica_n3 ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={async=fd77b6c8-e0aa-4c05-8dbc-b5c3a987226b166502595730377&qt=/admin/cores&coreNodeName=core_node4&collection.configName=confFaulty&name=backuprestore_testfailure_restored_shard1_replica_n3&action=CREATE&collection=backuprestore_testfailure_restored&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=0
   [junit4]   2> 221672 INFO  (parallelCoreAdminExecutor-2767-thread-1-processing-n:127.0.0.1:39251_solr x:backuprestore_testfailure_restored_shard1_replica_n3 fd77b6c8-e0aa-4c05-8dbc-b5c3a987226b166502595730377 CREATE) [n:127.0.0.1:39251_solr    x:backuprestore_testfailure_restored_shard1_replica_n3 ] o.a.s.h.a.CoreAdminOperation core create command async=fd77b6c8-e0aa-4c05-8dbc-b5c3a987226b166502595730377&qt=/admin/cores&coreNodeName=c

[...truncated too long message...]

(ssl, alpn, http/1.1, h2)}{127.0.0.1:0}
   [junit4]   2> 6141938 INFO  (jetty-closer-37990-thread-1) [     ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@3ce5a309{/solr,null,UNAVAILABLE}
   [junit4]   2> 6141938 INFO  (jetty-closer-37990-thread-1) [     ] o.e.j.s.session node0 Stopped scavenging
   [junit4]   2> 6141939 INFO  (SUITE-TestRequestRateLimiter-seed#[991D89747B2758D4]-worker) [     ] o.a.s.c.ZkTestServer Shutting down ZkTestServer.
   [junit4]   2> 6142142 WARN  (ZkTestServer Run Thread) [     ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	3	/solr/collections/c2/terms/shard1
   [junit4]   2> 	3	/solr/collections/c1/terms/shard1
   [junit4]   2> 	2	/solr/aliases.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	10	/solr/collections/c2/state.json
   [junit4]   2> 	10	/solr/collections/c1/state.json
   [junit4]   2> 	2	/solr/clusterprops.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	21	/solr/overseer/queue
   [junit4]   2> 	9	/solr/overseer/collection-queue-work
   [junit4]   2> 	6	/solr/collections
   [junit4]   2> 	4	/solr/live_nodes
   [junit4]   2> 
   [junit4]   2> 6142147 INFO  (SUITE-TestRequestRateLimiter-seed#[991D89747B2758D4]-worker) [     ] o.a.s.c.ZkTestServer waitForServerDown: 127.0.0.1:34469
   [junit4]   2> 6142147 INFO  (SUITE-TestRequestRateLimiter-seed#[991D89747B2758D4]-worker) [     ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:34469
   [junit4]   2> 6142147 INFO  (SUITE-TestRequestRateLimiter-seed#[991D89747B2758D4]-worker) [     ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 34469
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.servlet.TestRequestRateLimiter_991D89747B2758D4-001
   [junit4]   2> Aug 09, 2020 9:05:55 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene86): {}, docValues:{}, maxPointsInLeafNode=525, maxMBSortInHeap=7.083754016380093, sim=Asserting(RandomSimilarity(queryNorm=true): {}), locale=fr-TG, timezone=Africa/Luanda
   [junit4]   2> NOTE: Linux 4.15.0-112-generic amd64/Oracle Corporation 11.0.4 (64-bit)/cpus=8,threads=1,free=202423144,total=536870912
   [junit4]   2> NOTE: All tests run in this JVM: [SuggesterTest, DirectSolrConnectionTest, ScriptEngineTest, MetricsConfigTest, MetricsHistoryIntegrationTest, TestDocTermOrds, OverseerStatusTest, SolrCloudReportersTest, HDFSCollectionsAPITest, TestSolrTestCaseJ4, SolrCoreCheckLockOnStartupTest, DimensionalRoutedAliasUpdateProcessorTest, MoreLikeThisHandlerTest, TestSearcherReuse, CollectionsAPIDistributedZkTest, MetricsHistoryHandlerTest, TestNonDefinedSimilarityFactory, TestHdfsCloudBackupRestore, TestCorePropertiesReload, SaslZkACLProviderTest, TermsComponentTest, TestNumericTokenStream, SolrPluginUtilsTest, TestCursorMarkWithoutUniqueKey, TestConfigSets, SearchHandlerTest, SplitByPrefixTest, TestEmbeddedSolrServerAdminHandler, SimpleMLTQParserTest, TestNestedDocsSort, FieldMutatingUpdateProcessorTest, TestAnalyzedSuggestions, FacetPivotSmallTest, SmileWriterTest, BasicFunctionalityTest, TestSolrQueryResponse, DistanceUnitsTest, CollectionReloadTest, DateFieldTest, RandomizedTaggerTest, TestFieldCacheReopen, ForceLeaderWithTlogReplicasTest, TestCloudJSONFacetSKG, AddReplicaTest, SolrMetricReporterTest, TestSolrCloudWithHadoopAuthPlugin, TestUnifiedSolrHighlighterWithoutStoredId, TestRandomFaceting, TestSolrIndexConfig, MaxSizeAutoCommitTest, CoreAdminHandlerTest, SolrInfoBeanTest, UpdateParamsTest, DisMaxRequestHandlerTest, CdcrBootstrapTest, BlobRepositoryCloudTest, ClassificationUpdateProcessorTest, AtomicUpdateProcessorFactoryTest, TestReloadAndDeleteDocs, DeleteLastCustomShardedReplicaTest, TestRawResponseWriter, TestUseDocValuesAsStored, BJQFilterAccessibleTest, TestConfigReload, TestConfigSetsAPIExclusivity, TestQueryingOnDownCollection, SyncSliceTest, RequestHandlersTest, TestJmxIntegration, TestStressUserVersions, ZkControllerTest, TestStressLucene, QueryParsingTest, WordBreakSolrSpellCheckerTest, TestBadConfig, TestPhraseSuggestions, TestSchemalessBufferedUpdates, TestFoldingMultitermQuery, SchemaVersionSpecificBehaviorTest, TestStressInPlaceUpdates, TestFieldCacheVsDocValues, TestRandomCollapseQParserPlugin, TestReplicationHandlerDiskOverFlow, TestCaffeineCache, ResponseHeaderTest, HdfsRecoverLeaseTest, CachingDirectoryFactoryTest, IndexBasedSpellCheckerTest, TestLeaderElectionWithEmptyReplica, HdfsCollectionsAPIDistributedZkTest, DistributedFacetPivotLargeTest, TestWaitForStateWithJettyShutdowns, DeleteShardTest, MoveReplicaTest, TestXmlQParser, TestComplexPhraseQParserPlugin, PhrasesIdentificationComponentTest, SuggestComponentTest, TestIndexSearcher, HighlighterConfigTest, TestAtomicUpdateErrorCases, DistributedDebugComponentTest, ConcurrentCreateRoutedAliasTest, TestDynamicFieldNamesIndexCorrectly, URLClassifyProcessorTest, BufferStoreTest, TestRandomRequestDistribution, CustomHighlightComponentTest, TestJavabinTupleStreamParser, TestExportWriter, TestSortByMinMaxFunction, TestRestManager, TestDynamicFieldCollectionResource, HttpPartitionTest, TestMultiWordSynonyms, RulesTest, TestSolr4Spatial, TestFreeTextSuggestions, TestManagedSchemaAPI, TestSSLRandomization, RecoveryZkTest, TestLockTree, SegmentsInfoRequestHandlerTest, SharedFSAutoReplicaFailoverTest, SynonymTokenizerTest, TestDocSet, TestSchemaVersionResource, CategoryRoutedAliasUpdateProcessorTest, TestMultiValuedNumericRangeQuery, TestDistribDocBasedVersion, TestImplicitCoreProperties, ZkShardTermsTest, TestSolrCloudWithKerberosAlt, RootFieldTest, TestJsonFacetRefinement, NumberUtilsTest, TestLegacyFieldReuse, HighlighterWithoutStoredIdTest, TestPartialUpdateDeduplication, EchoParamsTest, MinimalSchemaTest, OutputWriterTest, TestDistributedMissingSort, TestDistributedSearch, ChaosMonkeyNothingIsSafeTest, ChaosMonkeyNothingIsSafeWithPullReplicasTest, CollectionStateZnodeTest, ConnectionManagerTest, CreateCollectionCleanupTest, CreateRoutedAliasTest, DistribDocExpirationUpdateProcessorTest, LeaderElectionTest, LeaderFailoverAfterPartitionTest, LeaderTragicEventTest, MigrateRouteKeyTest, MissingSegmentRecoveryTest, MultiThreadedOCPTest, NestedShardedAtomicUpdateTest, OverseerCollectionConfigSetProcessorTest, OverseerModifyCollectionTest, OverseerSolrResponseTest, OverseerTest, RemoteQueryErrorTest, ReplaceNodeTest, ReplicationFactorTest, RollingRestartTest, HdfsSyncSliceTest, TestClusterStateMutator, ZkCollectionPropsCachingTest, ZkStateReaderTest, ZkStateWriterTest, ImplicitSnitchTest, CoreSorterTest, DirectoryFactoryTest, ExitableDirectoryReaderTest, HdfsDirectoryFactoryTest, PluginInfoTest, QueryResultKeyTest, TestNRTOpen, TestQuerySenderListener, TestSimpleTextCodec, TestSolrDeletionPolicy1, TestXIncludeConfig, TestSolrCloudSnapshots, TestSolrCoreSnapshots, TestDistribPackageStore, FieldAnalysisRequestHandlerTest, TestHdfsBackupRestoreCore, TestReplicationHandler, XmlInterpolationTest, WrapperMergePolicyFactoryTest, CheckHdfsIndexTest, TestRemoteStreaming, TestStreamBody, TestUnInvertedFieldException, TestWriterPerf, TestMacroExpander, TestMacros, TestCSVResponseWriter, TestCustomDocTransformer, TestGeoJSONResponseWriter, TestChildDocTransformerHierarchy, TestExplainDocTransformer, TestSubQueryTransformer, TestSubQueryTransformerCrossCore, TestSubQueryTransformerDistrib, TestManagedResource, TestFieldCollectionResource, TestFieldResource, TestFieldTypeCollectionResource, TestFieldTypeResource, TestSchemaNameResource, TestSchemaResource, TestUniqueKeyFieldResource, TestManagedStopFilterFactory, TestManagedSynonymFilterFactory, TestManagedSynonymGraphFilterFactory, BadCopyFieldTest, BadIndexSchemaTest, BooleanFieldTest, ChangedSchemaMergeTest, CopyFieldTest, DocValuesMissingTest, DocValuesMultiTest, DocValuesTest, EnumFieldTest, ExternalFileFieldSortTest, IndexSchemaRuntimeFieldTest, IndexSchemaTest, ManagedSchemaRoundRobinCloudTest, NotRequiredUniqueKeyTest, NumericFieldsTest, OpenExchangeRatesOrgProviderTest, PolyFieldTest, PreAnalyzedFieldManagedSchemaCloudTest, PreAnalyzedFieldTest, PrimitiveFieldTypeTest, RankFieldTest, RequiredFieldsTest, ResolveAnalyzerByNameTest, TestBulkSchemaConcurrent, TestCloudManagedSchema, TestCloudSchemaless, TestCollationField, TestCollationFieldDocValues, TestHalfAndHalfDocValues, TestOmitPositions, TestPointFields, TestSchemaField, UUIDFieldTest, AnalyticsMergeStrategyTest, AnalyticsQueryTest, CurrencyRangeFacetCloudTest, CursorMarkTest, FuzzySearchTest, LargeFieldTest, MaxScoreCollectorTest, MergeStrategyTest, RankQueryTest, ReturnFieldsTest, SignificantTermsQParserPluginTest, SolrIndexSearcherTest, SortSpecParsingTest, SpatialFilterTest, TestExtendedDismaxParser, TestFieldSortValues, TestMinHashQParser, TestOverriddenPrefixQueryForCustomFieldType, TestPayloadScoreQParserPlugin, TestQueryTypes, TestReRankQParserPlugin, TestRecovery, TestSolrCachePerf, TestSolrCoreParser, TestSolrFieldCacheBean, TestSolrJ, TestSolrQueryParser, TestSurroundQueryParser, TestTermsQParserPlugin, TestTrieFacet, DistributedFacetSimpleRefinementLongTailTest, SpatialHeatmapFacetsTest, TestDistribIDF, TestExactSharedStatsCache, TestLRUStatsCache, AuditLoggerPluginTest, BasicAuthIntegrationTest, JWTAuthPluginTest, JWTVerificationkeyResolverTest, TestExternalRoleRuleBasedAuthorizationPlugin, TestPKIAuthenticationPlugin, TestSha256AuthenticationProvider, TestDelegationWithHadoopAuth, TestImpersonationWithHadoopAuth, TestRequestRateLimiter]
   [junit4] Completed [762/864 (3!)] on J1 in 4.80s, 2 tests, 1 failure <<< FAILURES!

[...truncated 52211 lines...]