You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2018/07/12 14:53:07 UTC

[JENKINS] Lucene-Solr-Tests-7.x - Build # 674 - Still Unstable

Build: https://builds.apache.org/job/Lucene-Solr-Tests-7.x/674/

7 tests failed.
FAILED:  org.apache.solr.cloud.LeaderTragicEventTest.testOtherReplicasAreNotActive

Error Message:
Jetty Connector is not open: -2

Stack Trace:
java.lang.IllegalStateException: Jetty Connector is not open: -2
	at __randomizedtesting.SeedInfo.seed([C47C02C47C793E78:41C82EB3408687E0]:0)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner.getBaseUrl(JettySolrRunner.java:499)
	at org.apache.solr.cloud.MiniSolrCloudCluster.getReplicaJetty(MiniSolrCloudCluster.java:539)
	at org.apache.solr.cloud.LeaderTragicEventTest.corruptLeader(LeaderTragicEventTest.java:100)
	at org.apache.solr.cloud.LeaderTragicEventTest.testOtherReplicasAreNotActive(LeaderTragicEventTest.java:150)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.LeaderTragicEventTest.test

Error Message:
Error from server at https://127.0.0.1:50149/solr: collection already exists: collection1

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:50149/solr: collection already exists: collection1
	at __randomizedtesting.SeedInfo.seed([C47C02C47C793E78:4C283D1ED2855380]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1106)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:886)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:819)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
	at org.apache.solr.cloud.LeaderTragicEventTest.test(LeaderTragicEventTest.java:70)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.autoscaling.HdfsAutoAddReplicasIntegrationTest.testSimple

Error Message:
Waiting for collection testSimple2 null Live Nodes: [127.0.0.1:35084_solr, 127.0.0.1:46677_solr] Last available state: DocCollection(testSimple2//collections/testSimple2/state.json/24)={   "pullReplicas":"0",   "replicationFactor":"2",   "shards":{     "shard1":{       "range":"80000000-ffffffff",       "state":"active",       "replicas":{         "core_node2":{           "dataDir":"hdfs://localhost:35476/data/testSimple2/core_node2/data/",           "base_url":"http://127.0.0.1:33253/solr",           "node_name":"127.0.0.1:33253_solr",           "type":"NRT",           "force_set_state":"false",           "ulogDir":"hdfs://localhost:35476/data/testSimple2/core_node2/data/tlog",           "core":"testSimple2_shard1_replica_n1",           "shared_storage":"true",           "state":"down"},         "core_node4":{           "dataDir":"hdfs://localhost:35476/data/testSimple2/core_node4/data/",           "base_url":"http://127.0.0.1:46677/solr",           "node_name":"127.0.0.1:46677_solr",           "type":"NRT",           "force_set_state":"false",           "ulogDir":"hdfs://localhost:35476/data/testSimple2/core_node4/data/tlog",           "core":"testSimple2_shard1_replica_n3",           "shared_storage":"true",           "state":"active",           "leader":"true"}}},     "shard2":{       "range":"0-7fffffff",       "state":"active",       "replicas":{         "core_node6":{           "dataDir":"hdfs://localhost:35476/data/testSimple2/core_node6/data/",           "base_url":"http://127.0.0.1:33253/solr",           "node_name":"127.0.0.1:33253_solr",           "type":"NRT",           "force_set_state":"false",           "ulogDir":"hdfs://localhost:35476/data/testSimple2/core_node6/data/tlog",           "core":"testSimple2_shard2_replica_n5",           "shared_storage":"true",           "state":"down"},         "core_node8":{           "dataDir":"hdfs://localhost:35476/data/testSimple2/core_node8/data/",           "base_url":"http://127.0.0.1:46677/solr",           "node_name":"127.0.0.1:46677_solr",           "type":"NRT",           "force_set_state":"false",           "ulogDir":"hdfs://localhost:35476/data/testSimple2/core_node8/data/tlog",           "core":"testSimple2_shard2_replica_n7",           "shared_storage":"true",           "state":"active",           "leader":"true"}}}},   "router":{"name":"compositeId"},   "maxShardsPerNode":"2",   "autoAddReplicas":"true",   "nrtReplicas":"2",   "tlogReplicas":"0"}

Stack Trace:
java.lang.AssertionError: Waiting for collection testSimple2
null
Live Nodes: [127.0.0.1:35084_solr, 127.0.0.1:46677_solr]
Last available state: DocCollection(testSimple2//collections/testSimple2/state.json/24)={
  "pullReplicas":"0",
  "replicationFactor":"2",
  "shards":{
    "shard1":{
      "range":"80000000-ffffffff",
      "state":"active",
      "replicas":{
        "core_node2":{
          "dataDir":"hdfs://localhost:35476/data/testSimple2/core_node2/data/",
          "base_url":"http://127.0.0.1:33253/solr",
          "node_name":"127.0.0.1:33253_solr",
          "type":"NRT",
          "force_set_state":"false",
          "ulogDir":"hdfs://localhost:35476/data/testSimple2/core_node2/data/tlog",
          "core":"testSimple2_shard1_replica_n1",
          "shared_storage":"true",
          "state":"down"},
        "core_node4":{
          "dataDir":"hdfs://localhost:35476/data/testSimple2/core_node4/data/",
          "base_url":"http://127.0.0.1:46677/solr",
          "node_name":"127.0.0.1:46677_solr",
          "type":"NRT",
          "force_set_state":"false",
          "ulogDir":"hdfs://localhost:35476/data/testSimple2/core_node4/data/tlog",
          "core":"testSimple2_shard1_replica_n3",
          "shared_storage":"true",
          "state":"active",
          "leader":"true"}}},
    "shard2":{
      "range":"0-7fffffff",
      "state":"active",
      "replicas":{
        "core_node6":{
          "dataDir":"hdfs://localhost:35476/data/testSimple2/core_node6/data/",
          "base_url":"http://127.0.0.1:33253/solr",
          "node_name":"127.0.0.1:33253_solr",
          "type":"NRT",
          "force_set_state":"false",
          "ulogDir":"hdfs://localhost:35476/data/testSimple2/core_node6/data/tlog",
          "core":"testSimple2_shard2_replica_n5",
          "shared_storage":"true",
          "state":"down"},
        "core_node8":{
          "dataDir":"hdfs://localhost:35476/data/testSimple2/core_node8/data/",
          "base_url":"http://127.0.0.1:46677/solr",
          "node_name":"127.0.0.1:46677_solr",
          "type":"NRT",
          "force_set_state":"false",
          "ulogDir":"hdfs://localhost:35476/data/testSimple2/core_node8/data/tlog",
          "core":"testSimple2_shard2_replica_n7",
          "shared_storage":"true",
          "state":"active",
          "leader":"true"}}}},
  "router":{"name":"compositeId"},
  "maxShardsPerNode":"2",
  "autoAddReplicas":"true",
  "nrtReplicas":"2",
  "tlogReplicas":"0"}
	at __randomizedtesting.SeedInfo.seed([C47C02C47C793E78:FCCF263A5B8AEAA9]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.SolrCloudTestCase.waitForState(SolrCloudTestCase.java:278)
	at org.apache.solr.cloud.autoscaling.AutoAddReplicasIntegrationTest.testSimple(AutoAddReplicasIntegrationTest.java:129)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.logging.TestLogWatcher.testLog4jWatcherRepeat

Error Message:


Stack Trace:
java.lang.AssertionError
	at org.junit.Assert.fail(Assert.java:92)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertTrue(Assert.java:54)
	at org.apache.solr.logging.TestLogWatcher.testLog4jWatcherRepeat(TestLogWatcher.java:69)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:45)
	at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:15)
	at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:42)
	at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:20)
	at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:28)
	at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:263)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:68)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:47)
	at org.junit.runners.ParentRunner$3.run(ParentRunner.java:231)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:60)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:229)
	at org.junit.runners.ParentRunner.access$000(ParentRunner.java:50)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:222)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:300)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:251)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)


FAILED:  org.apache.solr.logging.TestLogWatcher.testLog4jWatcher

Error Message:


Stack Trace:
java.lang.AssertionError
	at org.junit.Assert.fail(Assert.java:92)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertTrue(Assert.java:54)
	at org.apache.solr.logging.TestLogWatcher.testLog4jWatcher(TestLogWatcher.java:49)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:45)
	at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:15)
	at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:42)
	at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:20)
	at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:28)
	at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:263)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:68)
	at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:47)
	at org.junit.runners.ParentRunner$3.run(ParentRunner.java:231)
	at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:60)
	at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:229)
	at org.junit.runners.ParentRunner.access$000(ParentRunner.java:50)
	at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:222)
	at org.junit.runners.ParentRunner.run(ParentRunner.java:300)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:251)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)


FAILED:  org.apache.solr.metrics.reporters.SolrSlf4jReporterTest.testReporter

Error Message:
Test requires that log-level is at-least INFO, but INFO is disabled

Stack Trace:
java.lang.AssertionError: Test requires that log-level is at-least INFO, but INFO is disabled
	at __randomizedtesting.SeedInfo.seed([C47C02C47C793E78:9B982FF31775AD3D]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.metrics.reporters.SolrSlf4jReporterTest.ensureLoggingConfiguredAppropriately(SolrSlf4jReporterTest.java:100)
	at org.apache.solr.metrics.reporters.SolrSlf4jReporterTest.testReporter(SolrSlf4jReporterTest.java:49)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.update.processor.UpdateRequestProcessorFactoryTest.testUpdateDistribChainSkipping

Error Message:
Tests must be run with INFO level logging otherwise LogUpdateProcessor isn't used and can't be tested.

Stack Trace:
java.lang.AssertionError: Tests must be run with INFO level logging otherwise LogUpdateProcessor isn't used and can't be tested.
	at __randomizedtesting.SeedInfo.seed([C47C02C47C793E78:B598FC120A9219B4]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.update.processor.UpdateRequestProcessorFactoryTest.testUpdateDistribChainSkipping(UpdateRequestProcessorFactoryTest.java:101)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 12635 lines...]
   [junit4] Suite: org.apache.solr.metrics.reporters.SolrSlf4jReporterTest
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.metrics.reporters.SolrSlf4jReporterTest_C47C02C47C793E78-001/init-core-data-001
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=SolrSlf4jReporterTest -Dtests.method=testReporter -Dtests.seed=C47C02C47C793E78 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=th-TH-u-nu-thai-x-lvariant-TH -Dtests.timezone=Asia/Muscat -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] FAILURE 0.10s J1 | SolrSlf4jReporterTest.testReporter <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: Test requires that log-level is at-least INFO, but INFO is disabled
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([C47C02C47C793E78:9B982FF31775AD3D]:0)
   [junit4]    > 	at org.apache.solr.metrics.reporters.SolrSlf4jReporterTest.ensureLoggingConfiguredAppropriately(SolrSlf4jReporterTest.java:100)
   [junit4]    > 	at org.apache.solr.metrics.reporters.SolrSlf4jReporterTest.testReporter(SolrSlf4jReporterTest.java:49)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.metrics.reporters.SolrSlf4jReporterTest_C47C02C47C793E78-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=1272, maxMBSortInHeap=5.161103312303811, sim=RandomSimilarity(queryNorm=false): {}, locale=th-TH-u-nu-thai-x-lvariant-TH, timezone=Asia/Muscat
   [junit4]   2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_172 (64-bit)/cpus=4,threads=1,free=237482000,total=496500736
   [junit4]   2> NOTE: All tests run in this JVM: [TestSurroundQueryParser, DistributedMLTComponentTest, TestExportWriter, TestClassicSimilarityFactory, TestIntervalFaceting, AddSchemaFieldsUpdateProcessorFactoryTest, TestFieldCollectionResource, CheckHdfsIndexTest, TestReplicationHandlerBackup, TestExecutePlanAction, ZkNodePropsTest, LeaderElectionTest, TestReloadAndDeleteDocs, DistributedTermsComponentTest, HdfsSyncSliceTest, TestDistributedStatsComponentCardinality, ConjunctionSolrSpellCheckerTest, TestRequestStatusCollectionAPI, TestAnalyzedSuggestions, HighlighterTest, ProtectedTermFilterFactoryTest, TestLRUCache, DataDrivenBlockJoinTest, TestCollapseQParserPlugin, MinimalSchemaTest, TestDirectoryFactory, CurrencyRangeFacetCloudTest, TestCollationFieldDocValues, AssignTest, UtilsToolTest, TestLeaderElectionWithEmptyReplica, TestStressReorder, TestIBSimilarityFactory, SearchHandlerTest, SolrSlf4jReporterTest]
   [junit4] Completed [130/823 (1!)] on J1 in 0.25s, 1 test, 1 failure <<< FAILURES!

[...truncated 270 lines...]
   [junit4] Suite: org.apache.solr.update.processor.UpdateRequestProcessorFactoryTest
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.update.processor.UpdateRequestProcessorFactoryTest_C47C02C47C793E78-001/init-core-data-001
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=UpdateRequestProcessorFactoryTest -Dtests.method=testUpdateDistribChainSkipping -Dtests.seed=C47C02C47C793E78 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=es-PE -Dtests.timezone=America/Belem -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] FAILURE 0.24s J1 | UpdateRequestProcessorFactoryTest.testUpdateDistribChainSkipping <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: Tests must be run with INFO level logging otherwise LogUpdateProcessor isn't used and can't be tested.
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([C47C02C47C793E78:B598FC120A9219B4]:0)
   [junit4]    > 	at org.apache.solr.update.processor.UpdateRequestProcessorFactoryTest.testUpdateDistribChainSkipping(UpdateRequestProcessorFactoryTest.java:101)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.update.processor.UpdateRequestProcessorFactoryTest_C47C02C47C793E78-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=665, maxMBSortInHeap=5.712270958756288, sim=RandomSimilarity(queryNorm=false): {}, locale=es-PE, timezone=America/Belem
   [junit4]   2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_172 (64-bit)/cpus=4,threads=1,free=123358328,total=525860864
   [junit4]   2> NOTE: All tests run in this JVM: [TestSurroundQueryParser, DistributedMLTComponentTest, TestExportWriter, TestClassicSimilarityFactory, TestIntervalFaceting, AddSchemaFieldsUpdateProcessorFactoryTest, TestFieldCollectionResource, CheckHdfsIndexTest, TestReplicationHandlerBackup, TestExecutePlanAction, ZkNodePropsTest, LeaderElectionTest, TestReloadAndDeleteDocs, DistributedTermsComponentTest, HdfsSyncSliceTest, TestDistributedStatsComponentCardinality, ConjunctionSolrSpellCheckerTest, TestRequestStatusCollectionAPI, TestAnalyzedSuggestions, HighlighterTest, ProtectedTermFilterFactoryTest, TestLRUCache, DataDrivenBlockJoinTest, TestCollapseQParserPlugin, MinimalSchemaTest, TestDirectoryFactory, CurrencyRangeFacetCloudTest, TestCollationFieldDocValues, AssignTest, UtilsToolTest, TestLeaderElectionWithEmptyReplica, TestStressReorder, TestIBSimilarityFactory, SearchHandlerTest, SolrSlf4jReporterTest, PrimitiveFieldTypeTest, TestStressCloudBlindAtomicUpdates, TestNonDefinedSimilarityFactory, DocValuesNotIndexedTest, SaslZkACLProviderTest, JsonLoaderTest, TestClusterStateProvider, CdcrReplicationHandlerTest, TestUnInvertedFieldException, LIRRollingUpdatesTest, DirectoryFactoryTest, TestNRTOpen, MBeansHandlerTest, TestPhraseSuggestions, TestSolrConfigHandlerConcurrent, TestRawResponseWriter, UpdateRequestProcessorFactoryTest]
   [junit4] Completed [202/823 (2!)] on J1 in 1.66s, 3 tests, 1 failure <<< FAILURES!

[...truncated 471 lines...]
   [junit4] Suite: org.apache.solr.logging.TestLogWatcher
   [junit4] FAILURE 0.00s J2 | TestLogWatcher.testLog4jWatcherRepeat <<<
   [junit4]    > Throwable #1: java.lang.AssertionError
   [junit4]    > 	at org.apache.solr.logging.TestLogWatcher.testLog4jWatcherRepeat(TestLogWatcher.java:69)
   [junit4] FAILURE 0.00s J2 | TestLogWatcher.testLog4jWatcher <<<
   [junit4]    > Throwable #1: java.lang.AssertionError
   [junit4]    > 	at org.apache.solr.logging.TestLogWatcher.testLog4jWatcher(TestLogWatcher.java:49)
   [junit4] Completed [320/823 (3!)] on J2 in 0.00s, 2 tests, 2 failures <<< FAILURES!

[...truncated 526 lines...]
   [junit4] Suite: org.apache.solr.cloud.LeaderTragicEventTest
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderTragicEventTest_C47C02C47C793E78-001/init-core-data-001
   [junit4]   1> 14:00:55.800 [Thread-3023] ERROR org.apache.zookeeper.server.ZooKeeperServer - ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   1> 14:00:55.946 [jetty-launcher-4244-thread-1] ERROR org.apache.solr.util.StartupLoggingUtils - Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   1> 14:00:55.948 [jetty-launcher-4244-thread-2] ERROR org.apache.solr.util.StartupLoggingUtils - Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=LeaderTragicEventTest -Dtests.method=testOtherReplicasAreNotActive -Dtests.seed=C47C02C47C793E78 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=es-EC -Dtests.timezone=AET -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   3.65s J2 | LeaderTragicEventTest.testOtherReplicasAreNotActive <<<
   [junit4]    > Throwable #1: java.lang.IllegalStateException: Jetty Connector is not open: -2
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([C47C02C47C793E78:41C82EB3408687E0]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.embedded.JettySolrRunner.getBaseUrl(JettySolrRunner.java:499)
   [junit4]    > 	at org.apache.solr.cloud.MiniSolrCloudCluster.getReplicaJetty(MiniSolrCloudCluster.java:539)
   [junit4]    > 	at org.apache.solr.cloud.LeaderTragicEventTest.corruptLeader(LeaderTragicEventTest.java:100)
   [junit4]    > 	at org.apache.solr.cloud.LeaderTragicEventTest.testOtherReplicasAreNotActive(LeaderTragicEventTest.java:150)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   1> 14:01:00.165 [OverseerThreadFactory-6508-thread-2] ERROR org.apache.solr.cloud.api.collections.OverseerCollectionMessageHandler - Collection: collection1 operation: create failed:org.apache.solr.common.SolrException: collection already exists: collection1
   [junit4]   1> 	at org.apache.solr.cloud.api.collections.CreateCollectionCmd.call(CreateCollectionCmd.java:106)
   [junit4]   1> 	at org.apache.solr.cloud.api.collections.OverseerCollectionMessageHandler.processMessage(OverseerCollectionMessageHandler.java:255)
   [junit4]   1> 	at org.apache.solr.cloud.OverseerTaskProcessor$Runner.run(OverseerTaskProcessor.java:469)
   [junit4]   1> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)
   [junit4]   1> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   1> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   1> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   1> 
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=LeaderTragicEventTest -Dtests.method=test -Dtests.seed=C47C02C47C793E78 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=es-EC -Dtests.timezone=AET -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.02s J2 | LeaderTragicEventTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:50149/solr: collection already exists: collection1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([C47C02C47C793E78:4C283D1ED2855380]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1106)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:886)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:819)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
   [junit4]    > 	at org.apache.solr.cloud.LeaderTragicEventTest.test(LeaderTragicEventTest.java:70)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   1> 14:01:00.211 [SUITE-LeaderTragicEventTest-seed#[C47C02C47C793E78]-worker] ERROR org.apache.zookeeper.server.ZooKeeperServer - ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderTragicEventTest_C47C02C47C793E78-001
   [junit4]   2> Jul 12, 2018 2:01:00 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 3 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=874, maxMBSortInHeap=7.0664091663473965, sim=RandomSimilarity(queryNorm=false): {}, locale=es-EC, timezone=AET
   [junit4]   2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_172 (64-bit)/cpus=4,threads=1,free=199944320,total=514326528
   [junit4]   2> NOTE: All tests run in this JVM: [TestTrieFacet, DistributedQueryComponentCustomSortTest, TestGeoJSONResponseWriter, TestDFISimilarityFactory, TestRemoteStreaming, AtomicUpdateProcessorFactoryTest, TestFieldResource, TestReqParamsAPI, TestGenericDistributedQueue, ZkShardTermsTest, LeaderFailoverAfterPartitionTest, TestShardHandlerFactory, FacetPivotSmallTest, HdfsThreadLeakTest, TestExpandComponent, DirectSolrSpellCheckerTest, AutoAddReplicasIntegrationTest, TestBlendedInfixSuggestions, TestPostingsSolrHighlighter, TestCharFilters, TestLegacyNumericRangeQueryBuilder, DirectUpdateHandlerTest, TestComplexPhraseLeadingWildcard, OutputWriterTest, TestDynamicLoading, CursorMarkTest, TestHalfAndHalfDocValues, CollectionReloadTest, TestLeaderElectionZkExpiry, TestStressUserVersions, TestLMDirichletSimilarityFactory, ShufflingReplicaListTransformerTest, SolrCloudReportersTest, RequiredFieldsTest, TestStressInPlaceUpdates, TestPerFieldSimilarity, ForceLeaderTest, ShardRoutingCustomTest, MoreLikeThisHandlerTest, TestComputePlanAction, CdcrRequestHandlerTest, TestWriterPerf, LeaderElectionContextKeyTest, ExitableDirectoryReaderTest, TestQuerySenderListener, MetricsHandlerTest, BlockCacheTest, TestSystemCollAutoCreate, TestRawTransformer, CircularListTest, TestInfoStreamLogging, StatsReloadRaceTest, TestEmbeddedSolrServerAdminHandler, TestTrackingShardHandlerFactory, DistribCursorPagingTest, TestInitQParser, ClassificationUpdateProcessorIntegrationTest, MigrateRouteKeyTest, TestNodeLostTrigger, TestNumericTerms64, DistributedQueueTest, SolrJmxReporterCloudTest, SystemLogListenerTest, BlobRepositoryCloudTest, TestCorePropertiesReload, TestCSVResponseWriter, TestCloudSchemaless, TestChildDocTransformer, SpatialFilterTest, TestSolrCoreSnapshots, TestLegacyNumericUtils, TestExactSharedStatsCache, DateFieldTest, DistanceFunctionTest, TestDynamicFieldCollectionResource, TestClusterProperties, SpellingQueryConverterTest, HdfsRecoveryZkTest, TestCloudPseudoReturnFields, ClusterStateUpdateTest, TestExtendedDismaxParser, SolrMetricsIntegrationTest, HdfsDirectoryTest, TestXmlQParserPlugin, HdfsChaosMonkeySafeLeaderTest, JSONWriterTest, TestSolrQueryParser, ComputePlanActionTest, SolrCoreCheckLockOnStartupTest, TemplateUpdateProcessorTest, TestSearchPerf, UpdateParamsTest, ReplaceNodeTest, TestSolrXml, TestGraphTermsQParserPlugin, OverseerTest, PreAnalyzedUpdateProcessorTest, PeerSyncTest, BlockJoinFacetRandomTest, TestExclusionRuleCollectionAccess, TestMiniSolrCloudClusterSSL, TestUpdate, SolrIndexSplitterTest, IgnoreCommitOptimizeUpdateProcessorFactoryTest, RestoreTriggerStateTest, DeleteReplicaTest, TestScoreJoinQPScore, CoreAdminRequestStatusTest, TestSQLHandlerNonCloud, TestTrie, TestRecoveryHdfs, MetricTriggerTest, TestZkChroot, TestJettySolrRunner, CoreAdminCreateDiscoverTest, TestLazyCores, SolrTestCaseJ4Test, TestHdfsBackupRestoreCore, DistributedFacetPivotLargeTest, SecurityConfHandlerTest, TestCollectionAPI, PluginInfoTest, UUIDUpdateProcessorFallbackTest, ChaosMonkeySafeLeaderTest, TestSolrCLIRunExample, TestSimpleTrackingShardHandler, ConnectionManagerTest, QueryEqualityTest, TestSolrDeletionPolicy1, TestStressLucene, BasicFunctionalityTest, ConvertedLegacyTest, CursorPagingTest, DistributedIntervalFacetingTest, TestDistributedMissingSort, TestDocumentBuilder, TestHighlightDedupGrouping, TestJoin, PathHierarchyTokenizerFactoryTest, TestLuceneMatchVersion, TestReversedWildcardFilterFactory, TestWordDelimiterFilterFactory, TokenizerChainTest, TestEmbeddedSolrServerConstructors, ActionThrottleTest, AssignBackwardCompatibilityTest, BasicDistributedZkTest, BasicZkTest, ChaosMonkeySafeLeaderWithPullReplicasTest, CollectionsAPISolrJTest, LeaderTragicEventTest]
   [junit4] Completed [454/823 (4!)] on J2 in 5.02s, 2 tests, 2 errors <<< FAILURES!

[...truncated 355 lines...]
   [junit4] Suite: org.apache.solr.cloud.autoscaling.HdfsAutoAddReplicasIntegrationTest
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.HdfsAutoAddReplicasIntegrationTest_C47C02C47C793E78-001/init-core-data-001
   [junit4]   1> 14:10:01.838 [Thread-3863] ERROR org.apache.zookeeper.server.ZooKeeperServer - ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   1> 14:10:01.956 [jetty-launcher-8200-thread-1] ERROR org.apache.solr.util.StartupLoggingUtils - Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   1> 14:10:01.960 [jetty-launcher-8200-thread-3] ERROR org.apache.solr.util.StartupLoggingUtils - Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   1> 14:10:01.962 [jetty-launcher-8200-thread-2] ERROR org.apache.solr.util.StartupLoggingUtils - Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   1> 14:10:02.475 [OverseerAutoScalingTriggerThread-74132234243538951-127.0.0.1:35084_solr-n_0000000000] DEBUG org.apache.solr.cloud.autoscaling.OverseerTriggerThread - Adding .auto_add_replicas and .scheduled_maintenance triggers
   [junit4]   1> 14:10:02.477 [OverseerAutoScalingTriggerThread-74132234243538951-127.0.0.1:35084_solr-n_0000000000] DEBUG org.apache.solr.cloud.autoscaling.OverseerTriggerThread - Refreshing /autoscaling.json with znode version 1
   [junit4]   1> 14:10:02.477 [OverseerAutoScalingTriggerThread-74132234243538951-127.0.0.1:35084_solr-n_0000000000] DEBUG org.apache.solr.cloud.autoscaling.OverseerTriggerThread - Current znodeVersion 1, lastZnodeVersion -1
   [junit4]   1> 14:10:02.477 [OverseerAutoScalingTriggerThread-74132234243538951-127.0.0.1:35084_solr-n_0000000000] DEBUG org.apache.solr.cloud.autoscaling.OverseerTriggerThread - Processed trigger updates upto znodeVersion 1
   [junit4]   1> 14:10:02.532 [OverseerAutoScalingTriggerThread-74132234243538951-127.0.0.1:35084_solr-n_0000000000] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - NodeLostTrigger .auto_add_replicas - Initial livenodes: []
   [junit4]   1> 14:10:02.536 [ScheduledTrigger-7423-thread-1] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:10:02.566 [OverseerAutoScalingTriggerThread-74132234243538951-127.0.0.1:35084_solr-n_0000000000] DEBUG org.apache.solr.cloud.autoscaling.OverseerTriggerThread - -- clean old nodeAdded markers
   [junit4]   1> 14:10:02.566 [OverseerAutoScalingTriggerThread-74132234243538951-127.0.0.1:35084_solr-n_0000000000] DEBUG org.apache.solr.cloud.autoscaling.OverseerTriggerThread - Current znodeVersion 1, lastZnodeVersion 1
   [junit4]   1> Formatting using clusterid: testClusterID
   [junit4]   1> 14:10:03.543 [ScheduledTrigger-7423-thread-1] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
   [junit4]   1> 14:10:04.552 [ScheduledTrigger-7423-thread-3] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
   [junit4]   1> 14:10:04.940 [DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.HdfsAutoAddReplicasIntegrationTest_C47C02C47C793E78-001/tempDir-002/hdfsBaseDir/data/data1/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.HdfsAutoAddReplicasIntegrationTest_C47C02C47C793E78-001/tempDir-002/hdfsBaseDir/data/data2/]]  heartbeating to localhost/127.0.0.1:35476] ERROR org.apache.hadoop.hdfs.server.datanode.DirectoryScanner - dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 ms/sec. Assuming default value of 1000
   [junit4]   1> 14:10:05.552 [ScheduledTrigger-7423-thread-2] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
   [junit4]   1> 14:10:06.100 [DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.HdfsAutoAddReplicasIntegrationTest_C47C02C47C793E78-001/tempDir-002/hdfsBaseDir/data/data3/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.HdfsAutoAddReplicasIntegrationTest_C47C02C47C793E78-001/tempDir-002/hdfsBaseDir/data/data4/]]  heartbeating to localhost/127.0.0.1:35476] ERROR org.apache.hadoop.hdfs.server.datanode.DirectoryScanner - dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 ms/sec. Assuming default value of 1000
   [junit4]   1> 14:10:06.553 [ScheduledTrigger-7423-thread-2] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
   [junit4]   1> 14:10:07.553 [ScheduledTrigger-7423-thread-2] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
   [junit4]   1> 14:10:08.554 [ScheduledTrigger-7423-thread-2] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
   [junit4]   1> 14:10:12.425 [ScheduledTrigger-7423-thread-2] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
   [junit4]   1> 14:10:12.456 [Finalizer] ERROR org.apache.solr.core.CoreContainer - CoreContainer was not close prior to finalize(), indicates a bug -- POSSIBLE RESOURCE LEAK!!!  instance=1603358033
   [junit4]   1> 14:10:13.425 [ScheduledTrigger-7423-thread-4] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
   [junit4]   1> 14:10:14.427 [ScheduledTrigger-7423-thread-2] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
   [junit4]   1> 14:10:15.427 [ScheduledTrigger-7423-thread-4] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
   [junit4]   1> 14:10:16.428 [ScheduledTrigger-7423-thread-2] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
   [junit4]   1> 14:10:17.428 [ScheduledTrigger-7423-thread-2] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
   [junit4]   1> 14:10:18.441 [ScheduledTrigger-7423-thread-3] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
   [junit4]   1> 14:10:19.441 [ScheduledTrigger-7423-thread-3] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
   [junit4]   1> 14:10:20.441 [ScheduledTrigger-7423-thread-3] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
   [junit4]   1> 14:10:21.441 [ScheduledTrigger-7423-thread-3] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
   [junit4]   1> 14:10:22.443 [ScheduledTrigger-7423-thread-3] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
   [junit4]   1> 14:10:23.444 [ScheduledTrigger-7423-thread-3] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
   [junit4]   1> 14:10:24.444 [ScheduledTrigger-7423-thread-3] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
   [junit4]   1> 14:10:25.444 [ScheduledTrigger-7423-thread-3] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:10:25.444 [ScheduledTrigger-7423-thread-3] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Tracking lost node: 127.0.0.1:33253_solr
   [junit4]   1> 14:10:26.445 [ScheduledTrigger-7423-thread-1] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:10:27.445 [ScheduledTrigger-7423-thread-1] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:10:28.445 [ScheduledTrigger-7423-thread-1] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:10:29.446 [ScheduledTrigger-7423-thread-1] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:10:30.446 [ScheduledTrigger-7423-thread-1] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:10:31.446 [ScheduledTrigger-7423-thread-1] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:10:32.447 [ScheduledTrigger-7423-thread-3] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:10:33.447 [ScheduledTrigger-7423-thread-1] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:10:34.447 [ScheduledTrigger-7423-thread-4] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:10:35.448 [ScheduledTrigger-7423-thread-3] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:10:35.448 [ScheduledTrigger-7423-thread-3] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - NodeLostTrigger firing registered processor for lost nodes: [127.0.0.1:33253_solr]
   [junit4]   1> 14:10:35.450 [ScheduledTrigger-7423-thread-3] DEBUG org.apache.solr.cloud.autoscaling.ScheduledTriggers - ++++++++ Cooldown inactive - processing event: {
   [junit4]   1>   "id":"707770222e106bTedyclk091td2m5jo7m3vqel4b",
   [junit4]   1>   "source":".auto_add_replicas",
   [junit4]   1>   "eventTime":31656520885080171,
   [junit4]   1>   "eventType":"NODELOST",
   [junit4]   1>   "properties":{
   [junit4]   1>     "eventTimes":[31656520885080171],
   [junit4]   1>     "nodeNames":["127.0.0.1:33253_solr"]}}
   [junit4]   1> 14:10:35.451 [ScheduledTrigger-7423-thread-3] DEBUG org.apache.solr.cloud.autoscaling.ScheduledTriggers - Pausing all triggers: [.auto_add_replicas, .scheduled_maintenance]
   [junit4]   1> 14:10:35.453 [ScheduledTrigger-7423-thread-3] INFO  org.apache.solr.cloud.autoscaling.SystemLogListener - Collection .system does not exist, disabling logging.
   [junit4]   1> 14:10:35.461 [AutoscalingActionExecutor-7424-thread-1] DEBUG org.apache.solr.cloud.autoscaling.ScheduledTriggers - -- processing actions for {
   [junit4]   1>   "id":"707770222e106bTedyclk091td2m5jo7m3vqel4b",
   [junit4]   1>   "source":".auto_add_replicas",
   [junit4]   1>   "eventTime":31656520885080171,
   [junit4]   1>   "eventType":"NODELOST",
   [junit4]   1>   "properties":{
   [junit4]   1>     "eventTimes":[31656520885080171],
   [junit4]   1>     "_enqueue_time_":31656530891934584,
   [junit4]   1>     "nodeNames":["127.0.0.1:33253_solr"]}}
   [junit4]   1> 14:10:35.463 [AutoscalingActionExecutor-7424-thread-1] DEBUG org.apache.solr.cloud.autoscaling.ComputePlanAction - -- processing event: {
   [junit4]   1>   "id":"707770222e106bTedyclk091td2m5jo7m3vqel4b",
   [junit4]   1>   "source":".auto_add_replicas",
   [junit4]   1>   "eventTime":31656520885080171,
   [junit4]   1>   "eventType":"NODELOST",
   [junit4]   1>   "properties":{
   [junit4]   1>     "eventTimes":[31656520885080171],
   [junit4]   1>     "_enqueue_time_":31656530891934584,
   [junit4]   1>     "nodeNames":["127.0.0.1:33253_solr"]}} with context properties: {BEFORE_ACTION=[auto_add_replicas_plan]}
   [junit4]   1> 14:10:35.463 [AutoscalingActionExecutor-7424-thread-1] DEBUG org.apache.solr.client.solrj.cloud.autoscaling.PolicyHelper - Creating a new session
   [junit4]   1> 14:10:35.507 [AutoscalingActionExecutor-7424-thread-1] DEBUG org.apache.solr.client.solrj.cloud.autoscaling.PolicyHelper - New session created 
   [junit4]   1> 14:10:35.512 [AutoscalingActionExecutor-7424-thread-1] INFO  org.apache.solr.cloud.autoscaling.ComputePlanAction - Computed Plan: action=MOVEREPLICA&collection=testSimple1&targetNode=127.0.0.1:46677_solr&inPlaceMove=true&replica=core_node6
   [junit4]   1> 14:10:35.513 [AutoscalingActionExecutor-7424-thread-1] INFO  org.apache.solr.cloud.autoscaling.ComputePlanAction - Computed Plan: action=MOVEREPLICA&collection=testSimple1&targetNode=127.0.0.1:46677_solr&inPlaceMove=true&replica=core_node2
   [junit4]   1> 14:10:35.513 [AutoscalingActionExecutor-7424-thread-1] INFO  org.apache.solr.client.solrj.cloud.autoscaling.PolicyHelper - returnSession, curr-time 31656530954 sessionWrapper.createTime 31656530947771567, this.sessionWrapper.createTime 31656530947771567 
   [junit4]   1> 14:10:35.513 [AutoscalingActionExecutor-7424-thread-1] DEBUG org.apache.solr.client.solrj.cloud.autoscaling.PolicyHelper - session set to NULL
   [junit4]   1> 14:10:35.514 [AutoscalingActionExecutor-7424-thread-1] DEBUG org.apache.solr.cloud.autoscaling.ExecutePlanAction - -- processing event: {
   [junit4]   1>   "id":"707770222e106bTedyclk091td2m5jo7m3vqel4b",
   [junit4]   1>   "source":".auto_add_replicas",
   [junit4]   1>   "eventTime":31656520885080171,
   [junit4]   1>   "eventType":"NODELOST",
   [junit4]   1>   "properties":{
   [junit4]   1>     "eventTimes":[31656520885080171],
   [junit4]   1>     "_enqueue_time_":31656530891934584,
   [junit4]   1>     "nodeNames":["127.0.0.1:33253_solr"]}} with context properties: {operations=[org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@2e3f95d7, org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@21de2dfe], AFTER_ACTION=[auto_add_replicas_plan], BEFORE_ACTION=[auto_add_replicas_plan, execute_plan]}
   [junit4]   1> 14:10:35.514 [AutoscalingActionExecutor-7424-thread-1] DEBUG org.apache.solr.cloud.autoscaling.ExecutePlanAction - Executing operation: action=MOVEREPLICA&collection=testSimple1&targetNode=127.0.0.1:46677_solr&inPlaceMove=true&replica=core_node6
   [junit4]   1> 14:10:40.529 [AutoscalingActionExecutor-7424-thread-1] DEBUG org.apache.solr.cloud.autoscaling.ExecutePlanAction - Executing operation: action=MOVEREPLICA&collection=testSimple1&targetNode=127.0.0.1:46677_solr&inPlaceMove=true&replica=core_node2
   [junit4]   1> 14:10:42.327 [TEST-HdfsAutoAddReplicasIntegrationTest.testSimple-seed#[C47C02C47C793E78]] ERROR org.apache.solr.util.StartupLoggingUtils - Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   1> 14:10:42.589 [coreLoadExecutor-7516-thread-2-processing-n:127.0.0.1:33253_solr] ERROR org.apache.solr.cloud.CloudUtil - 
   [junit4]   1> org.apache.solr.common.SolrException: Will not load SolrCore testSimple1_shard2_replica_n5 because it has been replaced due to failover.
   [junit4]   1> 	at org.apache.solr.cloud.CloudUtil.checkSharedFSFailoverReplaced(CloudUtil.java:87) ~[java/:?]
   [junit4]   1> 	at org.apache.solr.cloud.ZkController.throwErrorIfReplicaReplaced(ZkController.java:2504) ~[java/:?]
   [junit4]   1> 	at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:678) ~[java/:?]
   [junit4]   1> 	at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197) ~[metrics-core-3.2.6.jar:3.2.6]
   [junit4]   1> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_172]
   [junit4]   1> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) ~[java/:?]
   [junit4]   1> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_172]
   [junit4]   1> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_172]
   [junit4]   1> 	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_172]
   [junit4]   1> 14:10:42.590 [coreLoadExecutor-7516-thread-1-processing-n:127.0.0.1:33253_solr] ERROR org.apache.solr.cloud.CloudUtil - 
   [junit4]   1> org.apache.solr.common.SolrException: Will not load SolrCore testSimple1_shard1_replica_n1 because it has been replaced due to failover.
   [junit4]   1> 	at org.apache.solr.cloud.CloudUtil.checkSharedFSFailoverReplaced(CloudUtil.java:87) ~[java/:?]
   [junit4]   1> 	at org.apache.solr.cloud.ZkController.throwErrorIfReplicaReplaced(ZkController.java:2504) ~[java/:?]
   [junit4]   1> 	at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:678) ~[java/:?]
   [junit4]   1> 	at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197) ~[metrics-core-3.2.6.jar:3.2.6]
   [junit4]   1> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_172]
   [junit4]   1> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) ~[java/:?]
   [junit4]   1> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_172]
   [junit4]   1> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_172]
   [junit4]   1> 	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_172]
   [junit4]   1> 14:10:42.591 [coreContainerWorkExecutor-7509-thread-1-processing-n:127.0.0.1:33253_solr] ERROR org.apache.solr.core.CoreContainer - Error waiting for SolrCore to be loaded on startup
   [junit4]   1> org.apache.solr.common.SolrException: Will not load SolrCore testSimple1_shard1_replica_n1 because it has been replaced due to failover.
   [junit4]   1> 	at org.apache.solr.cloud.CloudUtil.checkSharedFSFailoverReplaced(CloudUtil.java:90) ~[java/:?]
   [junit4]   1> 	at org.apache.solr.cloud.ZkController.throwErrorIfReplicaReplaced(ZkController.java:2504) ~[java/:?]
   [junit4]   1> 	at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:678) ~[java/:?]
   [junit4]   1> 	at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197) ~[metrics-core-3.2.6.jar:3.2.6]
   [junit4]   1> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_172]
   [junit4]   1> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) ~[java/:?]
   [junit4]   1> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_172]
   [junit4]   1> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_172]
   [junit4]   1> 	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_172]
   [junit4]   1> 14:10:42.592 [coreContainerWorkExecutor-7509-thread-1-processing-n:127.0.0.1:33253_solr] ERROR org.apache.solr.core.CoreContainer - Error waiting for SolrCore to be loaded on startup
   [junit4]   1> org.apache.solr.common.SolrException: Will not load SolrCore testSimple1_shard2_replica_n5 because it has been replaced due to failover.
   [junit4]   1> 	at org.apache.solr.cloud.CloudUtil.checkSharedFSFailoverReplaced(CloudUtil.java:90) ~[java/:?]
   [junit4]   1> 	at org.apache.solr.cloud.ZkController.throwErrorIfReplicaReplaced(ZkController.java:2504) ~[java/:?]
   [junit4]   1> 	at org.apache.solr.core.CoreContainer.lambda$load$13(CoreContainer.java:678) ~[java/:?]
   [junit4]   1> 	at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197) ~[metrics-core-3.2.6.jar:3.2.6]
   [junit4]   1> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_172]
   [junit4]   1> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) ~[java/:?]
   [junit4]   1> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_172]
   [junit4]   1> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_172]
   [junit4]   1> 	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_172]
   [junit4]   1> 14:10:44.937 [TEST-HdfsAutoAddReplicasIntegrationTest.testSimple-seed#[C47C02C47C793E78]] ERROR org.apache.solr.util.StartupLoggingUtils - Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   1> 14:10:45.574 [AutoscalingActionExecutor-7424-thread-1] DEBUG org.apache.solr.cloud.autoscaling.ScheduledTriggers - Resuming trigger: .auto_add_replicas after 5000ms
   [junit4]   1> 14:10:45.574 [AutoscalingActionExecutor-7424-thread-1] DEBUG org.apache.solr.cloud.autoscaling.ScheduledTriggers - Resuming trigger: .scheduled_maintenance after 5000ms
   [junit4]   1> 14:10:45.574 [AutoscalingActionExecutor-7424-thread-1] DEBUG org.apache.solr.cloud.autoscaling.ScheduledTriggers - -- processing took 10113 ms for event id=707770222e106bTedyclk091td2m5jo7m3vqel4b
   [junit4]   1> 14:10:50.297 [OldIndexDirectoryCleanupThreadForCore-testSimple2_shard2_replica_n5] ERROR org.apache.solr.core.HdfsDirectoryFactory - Error checking if hdfs path hdfs://localhost:35476/data/testSimple2/core_node6/data/ exists
   [junit4]   1> java.io.IOException: Filesystem closed
   [junit4]   1> 	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:808) ~[hadoop-hdfs-2.7.4.jar:?]
   [junit4]   1> 	at org.apache.hadoop.hdfs.DFSClient.getFileInfo(DFSClient.java:2105) ~[hadoop-hdfs-2.7.4.jar:?]
   [junit4]   1> 	at org.apache.hadoop.hdfs.DistributedFileSystem$22.doCall(DistributedFileSystem.java:1305) ~[hadoop-hdfs-2.7.4.jar:?]
   [junit4]   1> 	at org.apache.hadoop.hdfs.DistributedFileSystem$22.doCall(DistributedFileSystem.java:1301) ~[hadoop-hdfs-2.7.4.jar:?]
   [junit4]   1> 	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.7.4.jar:?]
   [junit4]   1> 	at org.apache.hadoop.hdfs.DistributedFileSystem.getFileStatus(DistributedFileSystem.java:1317) ~[hadoop-hdfs-2.7.4.jar:?]
   [junit4]   1> 	at org.apache.hadoop.fs.FileSystem.exists(FileSystem.java:1426) ~[hadoop-common-2.7.4.jar:?]
   [junit4]   1> 	at org.apache.solr.core.HdfsDirectoryFactory.cleanupOldIndexDirectories(HdfsDirectoryFactory.java:515) ~[java/:?]
   [junit4]   1> 	at org.apache.solr.core.SolrCore.lambda$cleanupOldIndexDirectories$21(SolrCore.java:3099) ~[java/:?]
   [junit4]   1> 	at java.lang.Thread.run(Thread.java:748) [?:1.8.0_172]
   [junit4]   1> 14:10:50.574 [ScheduledTrigger-7423-thread-4] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:10:51.575 [ScheduledTrigger-7423-thread-1] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:10:52.575 [ScheduledTrigger-7423-thread-4] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:10:53.575 [ScheduledTrigger-7423-thread-2] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:10:54.575 [ScheduledTrigger-7423-thread-2] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:10:55.575 [ScheduledTrigger-7423-thread-3] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:10:56.575 [ScheduledTrigger-7423-thread-4] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:10:57.576 [ScheduledTrigger-7423-thread-4] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:10:58.577 [ScheduledTrigger-7423-thread-4] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:10:59.577 [ScheduledTrigger-7423-thread-4] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:11:00.578 [ScheduledTrigger-7423-thread-1] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:11:01.578 [ScheduledTrigger-7423-thread-3] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:11:02.578 [ScheduledTrigger-7423-thread-1] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:11:03.578 [ScheduledTrigger-7423-thread-3] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:11:04.578 [ScheduledTrigger-7423-thread-1] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:11:05.579 [ScheduledTrigger-7423-thread-4] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:11:06.579 [ScheduledTrigger-7423-thread-1] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:11:07.579 [ScheduledTrigger-7423-thread-2] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:11:08.580 [ScheduledTrigger-7423-thread-1] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:11:09.580 [ScheduledTrigger-7423-thread-4] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:11:10.580 [ScheduledTrigger-7423-thread-3] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:11:11.580 [ScheduledTrigger-7423-thread-3] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:11:12.581 [ScheduledTrigger-7423-thread-3] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:11:13.581 [ScheduledTrigger-7423-thread-3] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:11:14.581 [ScheduledTrigger-7423-thread-3] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:11:15.581 [ScheduledTrigger-7423-thread-3] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:11:16.582 [ScheduledTrigger-7423-thread-2] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:11:17.582 [ScheduledTrigger-7423-thread-2] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:11:18.582 [ScheduledTrigger-7423-thread-2] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:11:19.582 [ScheduledTrigger-7423-thread-2] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:11:20.583 [ScheduledTrigger-7423-thread-2] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:11:21.583 [ScheduledTrigger-7423-thread-2] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:11:22.583 [ScheduledTrigger-7423-thread-2] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:11:23.584 [ScheduledTrigger-7423-thread-2] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:11:24.584 [ScheduledTrigger-7423-thread-2] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:11:25.584 [ScheduledTrigger-7423-thread-2] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:11:26.585 [ScheduledTrigger-7423-thread-2] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:11:27.585 [ScheduledTrigger-7423-thread-2] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:11:28.585 [ScheduledTrigger-7423-thread-2] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:11:29.585 [ScheduledTrigger-7423-thread-2] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:11:30.586 [ScheduledTrigger-7423-thread-2] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:11:31.586 [ScheduledTrigger-7423-thread-2] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:11:32.586 [ScheduledTrigger-7423-thread-2] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:11:33.587 [ScheduledTrigger-7423-thread-2] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:11:34.587 [ScheduledTrigger-7423-thread-2] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:11:35.587 [ScheduledTrigger-7423-thread-2] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:11:36.587 [ScheduledTrigger-7423-thread-4] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:11:37.588 [ScheduledTrigger-7423-thread-4] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:11:38.588 [ScheduledTrigger-7423-thread-4] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:11:39.588 [ScheduledTrigger-7423-thread-4] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:11:40.588 [ScheduledTrigger-7423-thread-4] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:11:41.589 [ScheduledTrigger-7423-thread-4] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:11:42.589 [ScheduledTrigger-7423-thread-4] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:11:43.589 [ScheduledTrigger-7423-thread-4] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:11:44.589 [ScheduledTrigger-7423-thread-4] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:11:45.590 [ScheduledTrigger-7423-thread-1] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:11:46.590 [ScheduledTrigger-7423-thread-1] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:11:47.590 [ScheduledTrigger-7423-thread-3] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:11:48.591 [ScheduledTrigger-7423-thread-3] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:11:49.591 [ScheduledTrigger-7423-thread-3] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:11:50.591 [ScheduledTrigger-7423-thread-3] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:11:51.591 [ScheduledTrigger-7423-thread-2] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:11:52.592 [ScheduledTrigger-7423-thread-4] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:11:53.592 [ScheduledTrigger-7423-thread-2] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:11:54.592 [ScheduledTrigger-7423-thread-3] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:11:55.592 [ScheduledTrigger-7423-thread-3] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:11:56.593 [ScheduledTrigger-7423-thread-3] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:11:57.593 [ScheduledTrigger-7423-thread-3] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:11:58.593 [ScheduledTrigger-7423-thread-3] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:11:59.593 [ScheduledTrigger-7423-thread-3] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:12:00.594 [ScheduledTrigger-7423-thread-3] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:12:01.594 [ScheduledTrigger-7423-thread-3] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:12:02.594 [ScheduledTrigger-7423-thread-3] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:12:03.594 [ScheduledTrigger-7423-thread-3] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:12:04.595 [ScheduledTrigger-7423-thread-3] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:12:05.595 [ScheduledTrigger-7423-thread-3] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:12:06.595 [ScheduledTrigger-7423-thread-3] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:12:07.595 [ScheduledTrigger-7423-thread-2] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:12:08.596 [ScheduledTrigger-7423-thread-2] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:12:09.596 [ScheduledTrigger-7423-thread-1] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:12:10.596 [ScheduledTrigger-7423-thread-1] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:12:11.596 [ScheduledTrigger-7423-thread-3] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:12:12.597 [ScheduledTrigger-7423-thread-3] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:12:13.597 [ScheduledTrigger-7423-thread-2] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:12:14.597 [ScheduledTrigger-7423-thread-2] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:12:15.597 [ScheduledTrigger-7423-thread-1] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:12:16.598 [ScheduledTrigger-7423-thread-2] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:12:17.598 [ScheduledTrigger-7423-thread-4] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:12:18.598 [ScheduledTrigger-7423-thread-2] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   1> 14:12:19.599 [ScheduledTrigger-7423-thread-2] DEBUG org.apache.solr.cloud.autoscaling.NodeLostTrigger - Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=HdfsAutoAddReplicasIntegrationTest -Dtests.method=testSimple -Dtests.seed=C47C02C47C793E78 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=en-IE -Dtests.timezone=Africa/Harare -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] FAILURE  133s J0 | HdfsAutoAddReplicasIntegrationTest.testSimple <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: Waiting for collection testSimple2
   [junit4]    > null
   [junit4]    > Live Nodes: [127.0.0.1:35084_solr, 127.0.0.1:46677_solr]
   [junit4]    > Last available state: DocCollection(testSimple2//collections/testSimple2/state.json/24)={
   [junit4]    >   "pullReplicas":"0",
   [junit4]    >   "replicationFactor":"2",
   [junit4]    >   "shards":{
   [junit4]    >     "shard1":{
   [junit4]    >       "range":"80000000-ffffffff",
   [junit4]    >       "state":"active",
   [junit4]    >       "replicas":{
   [junit4]    >         "core_node2":{
   [junit4]    >           "dataD

[...truncated too long message...]

ck:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

jar-checksums:
    [mkdir] Created dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/null1391071510
     [copy] Copying 241 files to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/null1391071510
   [delete] Deleting directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/null1391071510

check-working-copy:
[ivy:cachepath] :: resolving dependencies :: org.eclipse.jgit#org.eclipse.jgit-caller;working
[ivy:cachepath] 	confs: [default]
[ivy:cachepath] 	found org.eclipse.jgit#org.eclipse.jgit;4.6.0.201612231935-r in public
[ivy:cachepath] 	found com.jcraft#jsch;0.1.53 in public
[ivy:cachepath] 	found com.googlecode.javaewah#JavaEWAH;1.1.6 in public
[ivy:cachepath] 	found org.apache.httpcomponents#httpclient;4.3.6 in public
[ivy:cachepath] 	found org.apache.httpcomponents#httpcore;4.3.3 in public
[ivy:cachepath] 	found commons-logging#commons-logging;1.1.3 in public
[ivy:cachepath] 	found commons-codec#commons-codec;1.6 in public
[ivy:cachepath] 	found org.slf4j#slf4j-api;1.7.2 in public
[ivy:cachepath] :: resolution report :: resolve 52ms :: artifacts dl 1ms
	---------------------------------------------------------------------
	|                  |            modules            ||   artifacts   |
	|       conf       | number| search|dwnlded|evicted|| number|dwnlded|
	---------------------------------------------------------------------
	|      default     |   8   |   0   |   0   |   0   ||   8   |   0   |
	---------------------------------------------------------------------
[wc-checker] Initializing working copy...
[wc-checker] SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
[wc-checker] SLF4J: Defaulting to no-operation (NOP) logger implementation
[wc-checker] SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
[wc-checker] Checking working copy status...

-jenkins-base:

BUILD SUCCESSFUL
Total time: 130 minutes 34 seconds
Archiving artifacts
java.lang.InterruptedException: no matches found within 10000
	at hudson.FilePath$34.hasMatch(FilePath.java:2678)
	at hudson.FilePath$34.invoke(FilePath.java:2557)
	at hudson.FilePath$34.invoke(FilePath.java:2547)
	at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2918)
Also:   hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene
		at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1741)
		at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357)
		at hudson.remoting.Channel.call(Channel.java:955)
		at hudson.FilePath.act(FilePath.java:1036)
		at hudson.FilePath.act(FilePath.java:1025)
		at hudson.FilePath.validateAntFileMask(FilePath.java:2547)
		at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
		at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
		at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
		at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
		at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
		at hudson.model.Build$BuildExecution.post2(Build.java:186)
		at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
		at hudson.model.Run.execute(Run.java:1819)
		at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
		at hudson.model.ResourceController.execute(ResourceController.java:97)
		at hudson.model.Executor.run(Executor.java:429)
Caused: hudson.FilePath$TunneledInterruptedException
	at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2920)
	at hudson.remoting.UserRequest.perform(UserRequest.java:212)
	at hudson.remoting.UserRequest.perform(UserRequest.java:54)
	at hudson.remoting.Request$2.run(Request.java:369)
	at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:748)
Caused: java.lang.InterruptedException: java.lang.InterruptedException: no matches found within 10000
	at hudson.FilePath.act(FilePath.java:1038)
	at hudson.FilePath.act(FilePath.java:1025)
	at hudson.FilePath.validateAntFileMask(FilePath.java:2547)
	at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
	at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
	at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
	at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
	at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
	at hudson.model.Build$BuildExecution.post2(Build.java:186)
	at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
	at hudson.model.Run.execute(Run.java:1819)
	at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
	at hudson.model.ResourceController.execute(ResourceController.java:97)
	at hudson.model.Executor.run(Executor.java:429)
No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/hs_err_pid*". Configuration error?
Recording test results
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)

[JENKINS] Lucene-Solr-Tests-7.x - Build # 675 - Still Unstable

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-7.x/675/

2 tests failed.
FAILED:  org.apache.solr.cloud.MoveReplicaHDFSTest.testFailedMove

Error Message:
No live SolrServers available to handle this request:[https://127.0.0.1:49096/solr/MoveReplicaHDFSTest_failed_coll_true, https://127.0.0.1:58907/solr/MoveReplicaHDFSTest_failed_coll_true]

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[https://127.0.0.1:49096/solr/MoveReplicaHDFSTest_failed_coll_true, https://127.0.0.1:58907/solr/MoveReplicaHDFSTest_failed_coll_true]
	at __randomizedtesting.SeedInfo.seed([2FDF0F981C3BB8D7:8512DC6AABE86D07]:0)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1106)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:886)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:993)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:819)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:942)
	at org.apache.solr.cloud.MoveReplicaTest.testFailedMove(MoveReplicaTest.java:288)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:58907/solr/MoveReplicaHDFSTest_failed_coll_true: no servers hosting shard: shard2
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:436)
	... 46 more


FAILED:  org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.testDelegationTokenRenew

Error Message:
expected:<200> but was:<403>

Stack Trace:
java.lang.AssertionError: expected:<200> but was:<403>
	at __randomizedtesting.SeedInfo.seed([2FDF0F981C3BB8D7:1844FB8624F76573]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.junit.Assert.assertEquals(Assert.java:456)
	at org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.renewDelegationToken(TestSolrCloudWithDelegationTokens.java:132)
	at org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.verifyDelegationTokenRenew(TestSolrCloudWithDelegationTokens.java:316)
	at org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.testDelegationTokenRenew(TestSolrCloudWithDelegationTokens.java:333)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 13874 lines...]
   [junit4] Suite: org.apache.solr.cloud.MoveReplicaHDFSTest
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.MoveReplicaHDFSTest_2FDF0F981C3BB8D7-001/init-core-data-001
   [junit4]   2> 1981909 WARN  (SUITE-MoveReplicaHDFSTest-seed#[2FDF0F981C3BB8D7]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=19 numCloses=19
   [junit4]   2> 1981910 INFO  (SUITE-MoveReplicaHDFSTest-seed#[2FDF0F981C3BB8D7]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 1981911 INFO  (SUITE-MoveReplicaHDFSTest-seed#[2FDF0F981C3BB8D7]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 1981912 INFO  (SUITE-MoveReplicaHDFSTest-seed#[2FDF0F981C3BB8D7]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 1981912 INFO  (SUITE-MoveReplicaHDFSTest-seed#[2FDF0F981C3BB8D7]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 4 servers in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.MoveReplicaHDFSTest_2FDF0F981C3BB8D7-001/tempDir-001
   [junit4]   2> 1981912 INFO  (SUITE-MoveReplicaHDFSTest-seed#[2FDF0F981C3BB8D7]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 1981912 INFO  (Thread-3041) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1981913 INFO  (Thread-3041) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 1981914 ERROR (Thread-3041) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 1982013 INFO  (SUITE-MoveReplicaHDFSTest-seed#[2FDF0F981C3BB8D7]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:34585
   [junit4]   2> 1982019 INFO  (zkConnectionManagerCallback-3384-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1982027 INFO  (jetty-launcher-3381-thread-2) [    ] o.e.j.s.Server jetty-9.4.11.v20180605; built: 2018-06-05T18:24:03.829Z; git: d5fc0523cfa96bfebfbda19606cad384d772f04c; jvm 1.8.0_172-b11
   [junit4]   2> 1982028 INFO  (jetty-launcher-3381-thread-3) [    ] o.e.j.s.Server jetty-9.4.11.v20180605; built: 2018-06-05T18:24:03.829Z; git: d5fc0523cfa96bfebfbda19606cad384d772f04c; jvm 1.8.0_172-b11
   [junit4]   2> 1982030 INFO  (jetty-launcher-3381-thread-4) [    ] o.e.j.s.Server jetty-9.4.11.v20180605; built: 2018-06-05T18:24:03.829Z; git: d5fc0523cfa96bfebfbda19606cad384d772f04c; jvm 1.8.0_172-b11
   [junit4]   2> 1982036 INFO  (jetty-launcher-3381-thread-1) [    ] o.e.j.s.Server jetty-9.4.11.v20180605; built: 2018-06-05T18:24:03.829Z; git: d5fc0523cfa96bfebfbda19606cad384d772f04c; jvm 1.8.0_172-b11
   [junit4]   2> 1982037 INFO  (jetty-launcher-3381-thread-2) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 1982037 INFO  (jetty-launcher-3381-thread-2) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 1982037 INFO  (jetty-launcher-3381-thread-2) [    ] o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 1982037 INFO  (jetty-launcher-3381-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7fab09c9{/solr,null,AVAILABLE}
   [junit4]   2> 1982038 INFO  (jetty-launcher-3381-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@25ff9266{SSL,[ssl, http/1.1]}{127.0.0.1:53772}
   [junit4]   2> 1982038 INFO  (jetty-launcher-3381-thread-2) [    ] o.e.j.s.Server Started @1982109ms
   [junit4]   2> 1982038 INFO  (jetty-launcher-3381-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=53772}
   [junit4]   2> 1982038 ERROR (jetty-launcher-3381-thread-2) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1982038 INFO  (jetty-launcher-3381-thread-2) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 1982038 INFO  (jetty-launcher-3381-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.5.0
   [junit4]   2> 1982038 INFO  (jetty-launcher-3381-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1982038 INFO  (jetty-launcher-3381-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1982038 INFO  (jetty-launcher-3381-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-07-12T23:27:04.925Z
   [junit4]   2> 1982040 INFO  (jetty-launcher-3381-thread-3) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 1982040 INFO  (jetty-launcher-3381-thread-3) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 1982040 INFO  (jetty-launcher-3381-thread-3) [    ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 1982040 INFO  (jetty-launcher-3381-thread-3) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@36425d51{/solr,null,AVAILABLE}
   [junit4]   2> 1982041 INFO  (jetty-launcher-3381-thread-3) [    ] o.e.j.s.AbstractConnector Started ServerConnector@6b0408a1{SSL,[ssl, http/1.1]}{127.0.0.1:49096}
   [junit4]   2> 1982041 INFO  (jetty-launcher-3381-thread-3) [    ] o.e.j.s.Server Started @1982112ms
   [junit4]   2> 1982041 INFO  (jetty-launcher-3381-thread-3) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=49096}
   [junit4]   2> 1982041 ERROR (jetty-launcher-3381-thread-3) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1982041 INFO  (jetty-launcher-3381-thread-3) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 1982041 INFO  (jetty-launcher-3381-thread-3) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.5.0
   [junit4]   2> 1982041 INFO  (jetty-launcher-3381-thread-3) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1982041 INFO  (jetty-launcher-3381-thread-3) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1982041 INFO  (jetty-launcher-3381-thread-3) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-07-12T23:27:04.928Z
   [junit4]   2> 1982048 INFO  (jetty-launcher-3381-thread-1) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 1982049 INFO  (jetty-launcher-3381-thread-1) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 1982049 INFO  (jetty-launcher-3381-thread-1) [    ] o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 1982049 INFO  (jetty-launcher-3381-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3f6f25cf{/solr,null,AVAILABLE}
   [junit4]   2> 1982049 INFO  (jetty-launcher-3381-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@7f54135c{SSL,[ssl, http/1.1]}{127.0.0.1:36406}
   [junit4]   2> 1982049 INFO  (jetty-launcher-3381-thread-1) [    ] o.e.j.s.Server Started @1982120ms
   [junit4]   2> 1982049 INFO  (jetty-launcher-3381-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=36406}
   [junit4]   2> 1982050 ERROR (jetty-launcher-3381-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1982050 INFO  (jetty-launcher-3381-thread-1) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 1982050 INFO  (jetty-launcher-3381-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.5.0
   [junit4]   2> 1982050 INFO  (jetty-launcher-3381-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1982050 INFO  (jetty-launcher-3381-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1982050 INFO  (jetty-launcher-3381-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-07-12T23:27:04.937Z
   [junit4]   2> 1982052 INFO  (jetty-launcher-3381-thread-4) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 1982052 INFO  (jetty-launcher-3381-thread-4) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 1982052 INFO  (jetty-launcher-3381-thread-4) [    ] o.e.j.s.session node0 Scavenging every 600000ms
   [junit4]   2> 1982053 INFO  (jetty-launcher-3381-thread-4) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1434bfb7{/solr,null,AVAILABLE}
   [junit4]   2> 1982053 INFO  (jetty-launcher-3381-thread-4) [    ] o.e.j.s.AbstractConnector Started ServerConnector@28a150c9{SSL,[ssl, http/1.1]}{127.0.0.1:58907}
   [junit4]   2> 1982053 INFO  (jetty-launcher-3381-thread-4) [    ] o.e.j.s.Server Started @1982124ms
   [junit4]   2> 1982053 INFO  (jetty-launcher-3381-thread-4) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=58907}
   [junit4]   2> 1982053 ERROR (jetty-launcher-3381-thread-4) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1982053 INFO  (jetty-launcher-3381-thread-4) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 1982053 INFO  (jetty-launcher-3381-thread-4) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.5.0
   [junit4]   2> 1982053 INFO  (jetty-launcher-3381-thread-4) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1982053 INFO  (jetty-launcher-3381-thread-4) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1982053 INFO  (jetty-launcher-3381-thread-4) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-07-12T23:27:04.940Z
   [junit4]   2> 1982061 INFO  (zkConnectionManagerCallback-3388-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1982062 INFO  (jetty-launcher-3381-thread-3) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1982092 INFO  (zkConnectionManagerCallback-3386-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1982093 INFO  (jetty-launcher-3381-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1982097 INFO  (zkConnectionManagerCallback-3392-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1982097 INFO  (jetty-launcher-3381-thread-4) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1982102 INFO  (zkConnectionManagerCallback-3390-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1982103 INFO  (jetty-launcher-3381-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1982380 INFO  (jetty-launcher-3381-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:34585/solr
   [junit4]   2> 1982385 INFO  (jetty-launcher-3381-thread-4) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:34585/solr
   [junit4]   2> 1982411 INFO  (jetty-launcher-3381-thread-3) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:34585/solr
   [junit4]   2> 1982412 INFO  (zkConnectionManagerCallback-3396-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1982418 INFO  (zkConnectionManagerCallback-3400-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1982425 INFO  (zkConnectionManagerCallback-3404-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1982433 INFO  (zkConnectionManagerCallback-3406-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1982433 INFO  (zkConnectionManagerCallback-3408-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1982435 INFO  (zkConnectionManagerCallback-3410-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1982580 INFO  (jetty-launcher-3381-thread-2) [n:127.0.0.1:53772_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1982581 INFO  (jetty-launcher-3381-thread-4) [n:127.0.0.1:58907_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1982581 INFO  (jetty-launcher-3381-thread-2) [n:127.0.0.1:53772_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:53772_solr
   [junit4]   2> 1982582 INFO  (jetty-launcher-3381-thread-4) [n:127.0.0.1:58907_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:58907_solr
   [junit4]   2> 1982592 INFO  (jetty-launcher-3381-thread-3) [n:127.0.0.1:49096_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1982604 INFO  (zkCallback-3407-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1982606 INFO  (jetty-launcher-3381-thread-2) [n:127.0.0.1:53772_solr    ] o.a.s.c.Overseer Overseer (id=74134424650842120-127.0.0.1:53772_solr-n_0000000000) starting
   [junit4]   2> 1982607 INFO  (jetty-launcher-3381-thread-3) [n:127.0.0.1:49096_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1982609 INFO  (jetty-launcher-3381-thread-3) [n:127.0.0.1:49096_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 1982609 INFO  (jetty-launcher-3381-thread-3) [n:127.0.0.1:49096_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:49096_solr
   [junit4]   2> 1982614 INFO  (jetty-launcher-3381-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:34585/solr
   [junit4]   2> 1982625 INFO  (zkCallback-3405-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1982625 INFO  (zkCallback-3409-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1982626 INFO  (zkCallback-3407-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1982626 INFO  (zkConnectionManagerCallback-3421-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1982628 INFO  (zkConnectionManagerCallback-3426-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1982629 INFO  (jetty-launcher-3381-thread-4) [n:127.0.0.1:58907_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1982630 INFO  (jetty-launcher-3381-thread-4) [n:127.0.0.1:58907_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:34585/solr ready
   [junit4]   2> 1982630 INFO  (jetty-launcher-3381-thread-4) [n:127.0.0.1:58907_solr    ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 1982636 INFO  (zkConnectionManagerCallback-3435-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1982636 INFO  (zkConnectionManagerCallback-3430-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1982637 INFO  (jetty-launcher-3381-thread-2) [n:127.0.0.1:53772_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1982637 INFO  (jetty-launcher-3381-thread-3) [n:127.0.0.1:49096_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1982638 INFO  (jetty-launcher-3381-thread-2) [n:127.0.0.1:53772_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:34585/solr ready
   [junit4]   2> 1982638 INFO  (jetty-launcher-3381-thread-3) [n:127.0.0.1:49096_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:34585/solr ready
   [junit4]   2> 1982638 INFO  (jetty-launcher-3381-thread-3) [n:127.0.0.1:49096_solr    ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 1982659 INFO  (jetty-launcher-3381-thread-2) [n:127.0.0.1:53772_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 1982659 INFO  (jetty-launcher-3381-thread-2) [n:127.0.0.1:53772_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:53772_solr
   [junit4]   2> 1982660 INFO  (zkCallback-3407-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1982667 INFO  (jetty-launcher-3381-thread-3) [n:127.0.0.1:49096_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_49096.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2f7bf3f3
   [junit4]   2> 1982672 INFO  (jetty-launcher-3381-thread-4) [n:127.0.0.1:58907_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_58907.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2f7bf3f3
   [junit4]   2> 1982678 INFO  (jetty-launcher-3381-thread-3) [n:127.0.0.1:49096_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_49096.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2f7bf3f3
   [junit4]   2> 1982678 INFO  (jetty-launcher-3381-thread-3) [n:127.0.0.1:49096_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_49096.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2f7bf3f3
   [junit4]   2> 1982679 INFO  (jetty-launcher-3381-thread-3) [n:127.0.0.1:49096_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.MoveReplicaHDFSTest_2FDF0F981C3BB8D7-001/tempDir-001/node3/.
   [junit4]   2> 1982679 INFO  (zkConnectionManagerCallback-3438-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1982683 INFO  (jetty-launcher-3381-thread-1) [n:127.0.0.1:36406_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
   [junit4]   2> 1982684 INFO  (jetty-launcher-3381-thread-1) [n:127.0.0.1:36406_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1982684 INFO  (zkCallback-3409-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1982684 INFO  (zkCallback-3405-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1982692 INFO  (jetty-launcher-3381-thread-1) [n:127.0.0.1:36406_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 1982693 INFO  (jetty-launcher-3381-thread-1) [n:127.0.0.1:36406_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:36406_solr
   [junit4]   2> 1982693 INFO  (zkCallback-3409-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 1982693 INFO  (zkCallback-3407-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 1982696 INFO  (zkCallback-3405-thread-2) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 1982701 INFO  (jetty-launcher-3381-thread-4) [n:127.0.0.1:58907_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_58907.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2f7bf3f3
   [junit4]   2> 1982701 INFO  (jetty-launcher-3381-thread-4) [n:127.0.0.1:58907_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_58907.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2f7bf3f3
   [junit4]   2> 1982702 INFO  (jetty-launcher-3381-thread-4) [n:127.0.0.1:58907_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.MoveReplicaHDFSTest_2FDF0F981C3BB8D7-001/tempDir-001/node4/.
   [junit4]   2> 1982702 INFO  (jetty-launcher-3381-thread-2) [n:127.0.0.1:53772_solr    ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 1982743 INFO  (zkCallback-3429-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1982743 INFO  (zkCallback-3425-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1982743 INFO  (zkCallback-3434-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (4)
   [junit4]   2> 1982748 INFO  (jetty-launcher-3381-thread-2) [n:127.0.0.1:53772_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_53772.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2f7bf3f3
   [junit4]   2> 1982751 INFO  (zkCallback-3437-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 1982751 INFO  (zkCallback-3425-thread-2) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 1982760 INFO  (zkCallback-3429-thread-2) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 1982770 INFO  (jetty-launcher-3381-thread-2) [n:127.0.0.1:53772_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_53772.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2f7bf3f3
   [junit4]   2> 1982770 INFO  (jetty-launcher-3381-thread-2) [n:127.0.0.1:53772_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_53772.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2f7bf3f3
   [junit4]   2> 1982771 INFO  (jetty-launcher-3381-thread-2) [n:127.0.0.1:53772_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.MoveReplicaHDFSTest_2FDF0F981C3BB8D7-001/tempDir-001/node2/.
   [junit4]   2> 1982799 INFO  (zkConnectionManagerCallback-3447-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1982799 INFO  (jetty-launcher-3381-thread-1) [n:127.0.0.1:36406_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 1982800 INFO  (jetty-launcher-3381-thread-1) [n:127.0.0.1:36406_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:34585/solr ready
   [junit4]   2> 1982803 INFO  (jetty-launcher-3381-thread-1) [n:127.0.0.1:36406_solr    ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 1982821 INFO  (jetty-launcher-3381-thread-1) [n:127.0.0.1:36406_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_36406.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2f7bf3f3
   [junit4]   2> 1982835 INFO  (jetty-launcher-3381-thread-1) [n:127.0.0.1:36406_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_36406.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2f7bf3f3
   [junit4]   2> 1982835 INFO  (jetty-launcher-3381-thread-1) [n:127.0.0.1:36406_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_36406.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2f7bf3f3
   [junit4]   2> 1982843 INFO  (jetty-launcher-3381-thread-1) [n:127.0.0.1:36406_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.MoveReplicaHDFSTest_2FDF0F981C3BB8D7-001/tempDir-001/node1/.
   [junit4]   2> 1982873 INFO  (zkConnectionManagerCallback-3450-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1982876 INFO  (zkConnectionManagerCallback-3455-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1982877 INFO  (SUITE-MoveReplicaHDFSTest-seed#[2FDF0F981C3BB8D7]-worker) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 1982878 INFO  (SUITE-MoveReplicaHDFSTest-seed#[2FDF0F981C3BB8D7]-worker) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:34585/solr ready
   [junit4]   2> 1982898 INFO  (qtp27370531-13908) [n:127.0.0.1:36406_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :overseerstatus with params action=OVERSEERSTATUS&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1982903 INFO  (qtp27370531-13908) [n:127.0.0.1:36406_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=OVERSEERSTATUS&wt=javabin&version=2} status=0 QTime=4
   [junit4]   2> 1982904 INFO  (OverseerCollectionConfigSetProcessor-74134424650842120-127.0.0.1:53772_solr-n_0000000000) [    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   1> Formatting using clusterid: testClusterID
   [junit4]   2> 1982960 WARN  (SUITE-MoveReplicaHDFSTest-seed#[2FDF0F981C3BB8D7]-worker) [    ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
   [junit4]   2> 1982972 WARN  (SUITE-MoveReplicaHDFSTest-seed#[2FDF0F981C3BB8D7]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 1982987 INFO  (SUITE-MoveReplicaHDFSTest-seed#[2FDF0F981C3BB8D7]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 1983029 INFO  (SUITE-MoveReplicaHDFSTest-seed#[2FDF0F981C3BB8D7]-worker) [    ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/hdfs to ./temp/Jetty_localhost_36751_hdfs____l4z7ne/webapp
   [junit4]   2> 1983506 INFO  (SUITE-MoveReplicaHDFSTest-seed#[2FDF0F981C3BB8D7]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:36751
   [junit4]   2> 1983673 WARN  (SUITE-MoveReplicaHDFSTest-seed#[2FDF0F981C3BB8D7]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 1983674 INFO  (SUITE-MoveReplicaHDFSTest-seed#[2FDF0F981C3BB8D7]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 1983693 INFO  (SUITE-MoveReplicaHDFSTest-seed#[2FDF0F981C3BB8D7]-worker) [    ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_54479_datanode____.pznuq/webapp
   [junit4]   2> 1984234 INFO  (SUITE-MoveReplicaHDFSTest-seed#[2FDF0F981C3BB8D7]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:54479
   [junit4]   2> 1984356 WARN  (SUITE-MoveReplicaHDFSTest-seed#[2FDF0F981C3BB8D7]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 1984357 INFO  (SUITE-MoveReplicaHDFSTest-seed#[2FDF0F981C3BB8D7]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 1984375 INFO  (SUITE-MoveReplicaHDFSTest-seed#[2FDF0F981C3BB8D7]-worker) [    ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_37439_datanode____xns3a5/webapp
   [junit4]   2> 1984607 ERROR (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.MoveReplicaHDFSTest_2FDF0F981C3BB8D7-001/tempDir-002/hdfsBaseDir/data/data1/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.MoveReplicaHDFSTest_2FDF0F981C3BB8D7-001/tempDir-002/hdfsBaseDir/data/data2/]]  heartbeating to localhost/127.0.0.1:46290) [    ] o.a.h.h.s.d.DirectoryScanner dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 ms/sec. Assuming default value of 1000
   [junit4]   2> 1984696 INFO  (Block report processor) [    ] BlockStateChange BLOCK* processReport 0x7095d12c80fea9: from storage DS-8c22646a-f3be-4c69-8c5a-15b7e8d62ff2 node DatanodeRegistration(127.0.0.1:57260, datanodeUuid=da219929-46b2-4f9b-940f-d61143743008, infoPort=60998, infoSecurePort=0, ipcPort=60387, storageInfo=lv=-56;cid=testClusterID;nsid=945050172;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
   [junit4]   2> 1984696 INFO  (Block report processor) [    ] BlockStateChange BLOCK* processReport 0x7095d12c80fea9: from storage DS-438a20a0-d113-4d8c-8039-c478658f4b5a node DatanodeRegistration(127.0.0.1:57260, datanodeUuid=da219929-46b2-4f9b-940f-d61143743008, infoPort=60998, infoSecurePort=0, ipcPort=60387, storageInfo=lv=-56;cid=testClusterID;nsid=945050172;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 1985000 INFO  (SUITE-MoveReplicaHDFSTest-seed#[2FDF0F981C3BB8D7]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:37439
   [junit4]   2> 1985230 ERROR (DataNode: [[[DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.MoveReplicaHDFSTest_2FDF0F981C3BB8D7-001/tempDir-002/hdfsBaseDir/data/data3/, [DISK]file:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.MoveReplicaHDFSTest_2FDF0F981C3BB8D7-001/tempDir-002/hdfsBaseDir/data/data4/]]  heartbeating to localhost/127.0.0.1:46290) [    ] o.a.h.h.s.d.DirectoryScanner dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 ms/sec. Assuming default value of 1000
   [junit4]   2> 1985242 INFO  (Block report processor) [    ] BlockStateChange BLOCK* processReport 0x7095d14d55bc58: from storage DS-a1322347-01b5-4f8e-a308-0de3817fb7b3 node DatanodeRegistration(127.0.0.1:45386, datanodeUuid=df721ef8-e974-445a-b820-be2628696d5a, infoPort=58052, infoSecurePort=0, ipcPort=47789, storageInfo=lv=-56;cid=testClusterID;nsid=945050172;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
   [junit4]   2> 1985242 INFO  (Block report processor) [    ] BlockStateChange BLOCK* processReport 0x7095d14d55bc58: from storage DS-b1871a5c-6f9a-48b7-86db-691023b33f51 node DatanodeRegistration(127.0.0.1:45386, datanodeUuid=df721ef8-e974-445a-b820-be2628696d5a, infoPort=58052, infoSecurePort=0, ipcPort=47789, storageInfo=lv=-56;cid=testClusterID;nsid=945050172;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 1985484 INFO  (TEST-MoveReplicaHDFSTest.testNormalFailedMove-seed#[2FDF0F981C3BB8D7]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testNormalFailedMove
   [junit4]   2> 1985485 INFO  (TEST-MoveReplicaHDFSTest.testNormalFailedMove-seed#[2FDF0F981C3BB8D7]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 1985490 INFO  (TEST-MoveReplicaHDFSTest.testNormalFailedMove-seed#[2FDF0F981C3BB8D7]) [    ] o.e.j.s.Server jetty-9.4.11.v20180605; built: 2018-06-05T18:24:03.829Z; git: d5fc0523cfa96bfebfbda19606cad384d772f04c; jvm 1.8.0_172-b11
   [junit4]   2> 1985491 INFO  (TEST-MoveReplicaHDFSTest.testNormalFailedMove-seed#[2FDF0F981C3BB8D7]) [    ] o.e.j.s.session DefaultSessionIdManager workerName=node0
   [junit4]   2> 1985491 INFO  (TEST-MoveReplicaHDFSTest.testNormalFailedMove-seed#[2FDF0F981C3BB8D7]) [    ] o.e.j.s.session No SessionScavenger set, using defaults
   [junit4]   2> 1985491 INFO  (TEST-MoveReplicaHDFSTest.testNormalFailedMove-seed#[2FDF0F981C3BB8D7]) [    ] o.e.j.s.session node0 Scavenging every 660000ms
   [junit4]   2> 1985491 INFO  (TEST-MoveReplicaHDFSTest.testNormalFailedMove-seed#[2FDF0F981C3BB8D7]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@605d5333{/solr,null,AVAILABLE}
   [junit4]   2> 1985492 INFO  (TEST-MoveReplicaHDFSTest.testNormalFailedMove-seed#[2FDF0F981C3BB8D7]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@f58e3ad{SSL,[ssl, http/1.1]}{127.0.0.1:58397}
   [junit4]   2> 1985492 INFO  (TEST-MoveReplicaHDFSTest.testNormalFailedMove-seed#[2FDF0F981C3BB8D7]) [    ] o.e.j.s.Server Started @1985563ms
   [junit4]   2> 1985492 INFO  (TEST-MoveReplicaHDFSTest.testNormalFailedMove-seed#[2FDF0F981C3BB8D7]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=58397}
   [junit4]   2> 1985492 ERROR (TEST-MoveReplicaHDFSTest.testNormalFailedMove-seed#[2FDF0F981C3BB8D7]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1985492 INFO  (TEST-MoveReplicaHDFSTest.testNormalFailedMove-seed#[2FDF0F981C3BB8D7]) [    ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
   [junit4]   2> 1985492 INFO  (TEST-MoveReplicaHDFSTest.testNormalFailedMove-seed#[2FDF0F981C3BB8D7]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.5.0
   [junit4]   2> 1985492 INFO  (TEST-MoveReplicaHDFSTest.testNormalFailedMove-seed#[2FDF0F981C3BB8D7]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1985492 INFO  (TEST-MoveReplicaHDFSTest.testNormalFailedMove-seed#[2FDF0F981C3BB8D7]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1985492 INFO  (TEST-MoveReplicaHDFSTest.testNormalFailedMove-seed#[2FDF0F981C3BB8D7]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2018-07-12T23:27:08.379Z
   [junit4]   2> 1985494 INFO  (zkConnectionManagerCallback-3459-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1985494 INFO  (TEST-MoveReplicaHDFSTest.testNormalFailedMove-seed#[2FDF0F981C3BB8D7]) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1985656 INFO  (TEST-MoveReplicaHDFSTest.testNormalFailedMove-seed#[2FDF0F981C3BB8D7]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:34585/solr
   [junit4]   2> 1985658 INFO  (zkConnectionManagerCallback-3463-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1985661 INFO  (zkConnectionManagerCallback-3465-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1985665 INFO  (TEST-MoveReplicaHDFSTest.testNormalFailedMove-seed#[2FDF0F981C3BB8D7]) [n:127.0.0.1:58397_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 1985666 INFO  (TEST-MoveReplicaHDFSTest.testNormalFailedMove-seed#[2FDF0F981C3BB8D7]) [n:127.0.0.1:58397_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1985668 INFO  (TEST-MoveReplicaHDFSTest.testNormalFailedMove-seed#[2FDF0F981C3BB8D7]) [n:127.0.0.1:58397_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 1985668 INFO  (TEST-MoveReplicaHDFSTest.testNormalFailedMove-seed#[2FDF0F981C3BB8D7]) [n:127.0.0.1:58397_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:58397_solr
   [junit4]   2> 1985669 INFO  (zkCallback-3429-thread-2) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 1985669 INFO  (zkCallback-3407-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 1985669 INFO  (zkCallback-3437-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 1985669 INFO  (zkCallback-3409-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 1985669 INFO  (zkCallback-3434-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 1985669 INFO  (zkCallback-3425-thread-2) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 1985669 INFO  (zkCallback-3405-thread-2) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 1985682 INFO  (zkCallback-3446-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 1985685 INFO  (zkCallback-3454-thread-2) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 1985686 INFO  (zkCallback-3454-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 1985688 INFO  (zkCallback-3464-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 1985698 INFO  (zkConnectionManagerCallback-3472-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1985699 INFO  (TEST-MoveReplicaHDFSTest.testNormalFailedMove-seed#[2FDF0F981C3BB8D7]) [n:127.0.0.1:58397_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (5)
   [junit4]   2> 1985699 INFO  (TEST-MoveReplicaHDFSTest.testNormalFailedMove-seed#[2FDF0F981C3BB8D7]) [n:127.0.0.1:58397_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:34585/solr ready
   [junit4]   2> 1985700 INFO  (TEST-MoveReplicaHDFSTest.testNormalFailedMove-seed#[2FDF0F981C3BB8D7]) [n:127.0.0.1:58397_solr    ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
   [junit4]   2> 1985715 INFO  (TEST-MoveReplicaHDFSTest.testNormalFailedMove-seed#[2FDF0F981C3BB8D7]) [n:127.0.0.1:58397_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_58397.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2f7bf3f3
   [junit4]   2> 1985727 INFO  (TEST-MoveReplicaHDFSTest.testNormalFailedMove-seed#[2FDF0F981C3BB8D7]) [n:127.0.0.1:58397_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_58397.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2f7bf3f3
   [junit4]   2> 1985727 INFO  (TEST-MoveReplicaHDFSTest.testNormalFailedMove-seed#[2FDF0F981C3BB8D7]) [n:127.0.0.1:58397_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_58397.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2f7bf3f3
   [junit4]   2> 1985729 INFO  (TEST-MoveReplicaHDFSTest.testNormalFailedMove-seed#[2FDF0F981C3BB8D7]) [n:127.0.0.1:58397_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.MoveReplicaHDFSTest_2FDF0F981C3BB8D7-001/tempDir-001/node5/.
   [junit4]   2> 1985811 INFO  (zkConnectionManagerCallback-3475-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 1985815 INFO  (qtp27370531-13906) [n:127.0.0.1:36406_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&autoAddReplicas=false&name=MoveReplicaHDFSTest_failed_coll_false&nrtReplicas=2&action=CREATE&numShards=2&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1985817 INFO  (OverseerThreadFactory-5871-thread-2) [    ] o.a.s.c.a.c.CreateCollectionCmd Create collection MoveReplicaHDFSTest_failed_coll_false
   [junit4]   2> 1985924 INFO  (OverseerStateUpdate-74134424650842120-127.0.0.1:53772_solr-n_0000000000) [    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"MoveReplicaHDFSTest_failed_coll_false",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:49096/solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 1985926 INFO  (OverseerStateUpdate-74134424650842120-127.0.0.1:53772_solr-n_0000000000) [    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"MoveReplicaHDFSTest_failed_coll_false",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n3",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:58397/solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 1985928 INFO  (OverseerStateUpdate-74134424650842120-127.0.0.1:53772_solr-n_0000000000) [    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"MoveReplicaHDFSTest_failed_coll_false",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n4",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:53772/solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 1985929 INFO  (OverseerStateUpdate-74134424650842120-127.0.0.1:53772_solr-n_0000000000) [    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"MoveReplicaHDFSTest_failed_coll_false",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n6",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:58907/solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 1986191 INFO  (qtp951356604-14185) [n:127.0.0.1:58397_solr    x:MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n3] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node5&collection.configName=conf1&newCollection=true&name=MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n3&action=CREATE&numShards=2&collection=MoveReplicaHDFSTest_failed_coll_false&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 1986191 INFO  (qtp163946297-13917) [n:127.0.0.1:58907_solr    x:MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n6] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node8&collection.configName=conf1&newCollection=true&name=MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n6&action=CREATE&numShards=2&collection=MoveReplicaHDFSTest_failed_coll_false&shard=shard2&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 1986191 INFO  (qtp163946297-13917) [n:127.0.0.1:58907_solr    x:MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n6] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 1986201 INFO  (qtp902377229-13895) [n:127.0.0.1:49096_solr    x:MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n1] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node2&collection.configName=conf1&newCollection=true&name=MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n1&action=CREATE&numShards=2&collection=MoveReplicaHDFSTest_failed_coll_false&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 1986202 INFO  (qtp722955117-13884) [n:127.0.0.1:53772_solr    x:MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n4] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node7&collection.configName=conf1&newCollection=true&name=MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n4&action=CREATE&numShards=2&collection=MoveReplicaHDFSTest_failed_coll_false&shard=shard2&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 1987203 INFO  (qtp951356604-14185) [n:127.0.0.1:58397_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard1 r:core_node5 x:MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n3] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.5.0
   [junit4]   2> 1987215 INFO  (qtp902377229-13895) [n:127.0.0.1:49096_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard1 r:core_node2 x:MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.5.0
   [junit4]   2> 1987219 INFO  (qtp722955117-13884) [n:127.0.0.1:53772_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard2 r:core_node7 x:MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n4] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.5.0
   [junit4]   2> 1987229 INFO  (qtp902377229-13895) [n:127.0.0.1:49096_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard1 r:core_node2 x:MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n1] o.a.s.s.IndexSchema [MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n1] Schema name=minimal
   [junit4]   2> 1987231 INFO  (qtp902377229-13895) [n:127.0.0.1:49096_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard1 r:core_node2 x:MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 1987231 INFO  (qtp902377229-13895) [n:127.0.0.1:49096_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard1 r:core_node2 x:MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n1' using configuration from collection MoveReplicaHDFSTest_failed_coll_false, trusted=true
   [junit4]   2> 1987232 INFO  (qtp902377229-13895) [n:127.0.0.1:49096_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard1 r:core_node2 x:MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_49096.solr.core.MoveReplicaHDFSTest_failed_coll_false.shard1.replica_n1' (registry 'solr.core.MoveReplicaHDFSTest_failed_coll_false.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2f7bf3f3
   [junit4]   2> 1987236 INFO  (qtp163946297-13917) [n:127.0.0.1:58907_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard2 r:core_node8 x:MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n6] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.5.0
   [junit4]   2> 1987239 INFO  (qtp902377229-13895) [n:127.0.0.1:49096_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard1 r:core_node2 x:MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:46290/data
   [junit4]   2> 1987239 INFO  (qtp902377229-13895) [n:127.0.0.1:49096_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard1 r:core_node2 x:MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
   [junit4]   2> 1987239 INFO  (qtp902377229-13895) [n:127.0.0.1:49096_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard1 r:core_node2 x:MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1987239 INFO  (qtp902377229-13895) [n:127.0.0.1:49096_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard1 r:core_node2 x:MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n1] o.a.s.c.SolrCore [[MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.MoveReplicaHDFSTest_2FDF0F981C3BB8D7-001/tempDir-001/node3/MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n1], dataDir=[hdfs://localhost:46290/data/MoveReplicaHDFSTest_failed_coll_false/core_node2/data/]
   [junit4]   2> 1987243 INFO  (qtp902377229-13895) [n:127.0.0.1:49096_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard1 r:core_node2 x:MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:46290/data/MoveReplicaHDFSTest_failed_coll_false/core_node2/data/snapshot_metadata
   [junit4]   2> 1987244 INFO  (qtp951356604-14185) [n:127.0.0.1:58397_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard1 r:core_node5 x:MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n3] o.a.s.s.IndexSchema [MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n3] Schema name=minimal
   [junit4]   2> 1987246 INFO  (qtp951356604-14185) [n:127.0.0.1:58397_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard1 r:core_node5 x:MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n3] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 1987246 INFO  (qtp951356604-14185) [n:127.0.0.1:58397_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard1 r:core_node5 x:MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n3] o.a.s.c.CoreContainer Creating SolrCore 'MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n3' using configuration from collection MoveReplicaHDFSTest_failed_coll_false, trusted=true
   [junit4]   2> 1987247 INFO  (qtp951356604-14185) [n:127.0.0.1:58397_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard1 r:core_node5 x:MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n3] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_58397.solr.core.MoveReplicaHDFSTest_failed_coll_false.shard1.replica_n3' (registry 'solr.core.MoveReplicaHDFSTest_failed_coll_false.shard1.replica_n3') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2f7bf3f3
   [junit4]   2> 1987247 INFO  (qtp951356604-14185) [n:127.0.0.1:58397_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard1 r:core_node5 x:MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n3] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:46290/data
   [junit4]   2> 1987247 INFO  (qtp951356604-14185) [n:127.0.0.1:58397_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard1 r:core_node5 x:MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n3] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
   [junit4]   2> 1987247 INFO  (qtp951356604-14185) [n:127.0.0.1:58397_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard1 r:core_node5 x:MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n3] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1987247 INFO  (qtp951356604-14185) [n:127.0.0.1:58397_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard1 r:core_node5 x:MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n3] o.a.s.c.SolrCore [[MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n3] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.MoveReplicaHDFSTest_2FDF0F981C3BB8D7-001/tempDir-001/node5/MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n3], dataDir=[hdfs://localhost:46290/data/MoveReplicaHDFSTest_failed_coll_false/core_node5/data/]
   [junit4]   2> 1987248 INFO  (qtp951356604-14185) [n:127.0.0.1:58397_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard1 r:core_node5 x:MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n3] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:46290/data/MoveReplicaHDFSTest_failed_coll_false/core_node5/data/snapshot_metadata
   [junit4]   2> 1987254 INFO  (qtp722955117-13884) [n:127.0.0.1:53772_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard2 r:core_node7 x:MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n4] o.a.s.s.IndexSchema [MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n4] Schema name=minimal
   [junit4]   2> 1987258 INFO  (qtp163946297-13917) [n:127.0.0.1:58907_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard2 r:core_node8 x:MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n6] o.a.s.s.IndexSchema [MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n6] Schema name=minimal
   [junit4]   2> 1987281 INFO  (qtp163946297-13917) [n:127.0.0.1:58907_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard2 r:core_node8 x:MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n6] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 1987281 INFO  (qtp163946297-13917) [n:127.0.0.1:58907_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard2 r:core_node8 x:MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n6] o.a.s.c.CoreContainer Creating SolrCore 'MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n6' using configuration from collection MoveReplicaHDFSTest_failed_coll_false, trusted=true
   [junit4]   2> 1987282 INFO  (qtp163946297-13917) [n:127.0.0.1:58907_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard2 r:core_node8 x:MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n6] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_58907.solr.core.MoveReplicaHDFSTest_failed_coll_false.shard2.replica_n6' (registry 'solr.core.MoveReplicaHDFSTest_failed_coll_false.shard2.replica_n6') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2f7bf3f3
   [junit4]   2> 1987282 INFO  (qtp163946297-13917) [n:127.0.0.1:58907_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard2 r:core_node8 x:MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n6] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:46290/data
   [junit4]   2> 1987282 INFO  (qtp163946297-13917) [n:127.0.0.1:58907_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard2 r:core_node8 x:MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n6] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
   [junit4]   2> 1987282 INFO  (qtp163946297-13917) [n:127.0.0.1:58907_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard2 r:core_node8 x:MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n6] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1987282 INFO  (qtp163946297-13917) [n:127.0.0.1:58907_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard2 r:core_node8 x:MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n6] o.a.s.c.SolrCore [[MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n6] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.MoveReplicaHDFSTest_2FDF0F981C3BB8D7-001/tempDir-001/node4/MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n6], dataDir=[hdfs://localhost:46290/data/MoveReplicaHDFSTest_failed_coll_false/core_node8/data/]
   [junit4]   2> 1987284 INFO  (qtp163946297-13917) [n:127.0.0.1:58907_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard2 r:core_node8 x:MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n6] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:46290/data/MoveReplicaHDFSTest_failed_coll_false/core_node8/data/snapshot_metadata
   [junit4]   2> 1987289 INFO  (qtp722955117-13884) [n:127.0.0.1:53772_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard2 r:core_node7 x:MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n4] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 1987290 INFO  (qtp722955117-13884) [n:127.0.0.1:53772_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard2 r:core_node7 x:MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n4] o.a.s.c.CoreContainer Creating SolrCore 'MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n4' using configuration from collection MoveReplicaHDFSTest_failed_coll_false, trusted=true
   [junit4]   2> 1987290 INFO  (qtp722955117-13884) [n:127.0.0.1:53772_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard2 r:core_node7 x:MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n4] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_53772.solr.core.MoveReplicaHDFSTest_failed_coll_false.shard2.replica_n4' (registry 'solr.core.MoveReplicaHDFSTest_failed_coll_false.shard2.replica_n4') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2f7bf3f3
   [junit4]   2> 1987290 INFO  (qtp722955117-13884) [n:127.0.0.1:53772_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard2 r:core_node7 x:MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n4] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost:46290/data
   [junit4]   2> 1987290 INFO  (qtp722955117-13884) [n:127.0.0.1:53772_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard2 r:core_node7 x:MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n4] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
   [junit4]   2> 1987290 INFO  (qtp722955117-13884) [n:127.0.0.1:53772_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard2 r:core_node7 x:MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n4] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1987290 INFO  (qtp722955117-13884) [n:127.0.0.1:53772_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard2 r:core_node7 x:MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n4] o.a.s.c.SolrCore [[MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n4] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.MoveReplicaHDFSTest_2FDF0F981C3BB8D7-001/tempDir-001/node2/MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n4], dataDir=[hdfs://localhost:46290/data/MoveReplicaHDFSTest_failed_coll_false/core_node7/data/]
   [junit4]   2> 1987291 INFO  (qtp722955117-13884) [n:127.0.0.1:53772_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard2 r:core_node7 x:MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n4] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:46290/data/MoveReplicaHDFSTest_failed_coll_false/core_node7/data/snapshot_metadata
   [junit4]   2> 1987686 INFO  (qtp902377229-13895) [n:127.0.0.1:49096_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard1 r:core_node2 x:MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:46290/data/MoveReplicaHDFSTest_failed_coll_false/core_node2/data
   [junit4]   2> 1987706 INFO  (qtp951356604-14185) [n:127.0.0.1:58397_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard1 r:core_node5 x:MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n3] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:46290/data/MoveReplicaHDFSTest_failed_coll_false/core_node5/data
   [junit4]   2> 1987721 INFO  (qtp163946297-13917) [n:127.0.0.1:58907_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard2 r:core_node8 x:MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n6] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:46290/data/MoveReplicaHDFSTest_failed_coll_false/core_node8/data
   [junit4]   2> 1987768 INFO  (qtp722955117-13884) [n:127.0.0.1:53772_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard2 r:core_node7 x:MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n4] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:46290/data/MoveReplicaHDFSTest_failed_coll_false/core_node7/data
   [junit4]   2> 1987845 INFO  (qtp163946297-13917) [n:127.0.0.1:58907_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard2 r:core_node8 x:MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n6] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:46290/data/MoveReplicaHDFSTest_failed_coll_false/core_node8/data/index
   [junit4]   2> 1987851 INFO  (qtp902377229-13895) [n:127.0.0.1:49096_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard1 r:core_node2 x:MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:46290/data/MoveReplicaHDFSTest_failed_coll_false/core_node2/data/index
   [junit4]   2> 1987854 INFO  (qtp951356604-14185) [n:127.0.0.1:58397_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard1 r:core_node5 x:MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n3] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:46290/data/MoveReplicaHDFSTest_failed_coll_false/core_node5/data/index
   [junit4]   2> 1987890 INFO  (qtp722955117-13884) [n:127.0.0.1:53772_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard2 r:core_node7 x:MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n4] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost:46290/data/MoveReplicaHDFSTest_failed_coll_false/core_node7/data/index
   [junit4]   2> 1988289 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:57260 is added to blk_1073741825_1001{UCState=COMMITTED, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-8c22646a-f3be-4c69-8c5a-15b7e8d62ff2:NORMAL:127.0.0.1:57260|RBW], ReplicaUC[[DISK]DS-a1322347-01b5-4f8e-a308-0de3817fb7b3:NORMAL:127.0.0.1:45386|RBW]]} size 69
   [junit4]   2> 1988290 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:45386 is added to blk_1073741825_1001 size 69
   [junit4]   2> 1988317 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:45386 is added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-8c22646a-f3be-4c69-8c5a-15b7e8d62ff2:NORMAL:127.0.0.1:57260|RBW], ReplicaUC[[DISK]DS-a1322347-01b5-4f8e-a308-0de3817fb7b3:NORMAL:127.0.0.1:45386|RBW]]} size 0
   [junit4]   2> 1988319 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:57260 is added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-8c22646a-f3be-4c69-8c5a-15b7e8d62ff2:NORMAL:127.0.0.1:57260|RBW], ReplicaUC[[DISK]DS-a1322347-01b5-4f8e-a308-0de3817fb7b3:NORMAL:127.0.0.1:45386|RBW]]} size 0
   [junit4]   2> 1988320 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:45386 is added to blk_1073741828_1004{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-438a20a0-d113-4d8c-8039-c478658f4b5a:NORMAL:127.0.0.1:57260|RBW], ReplicaUC[[DISK]DS-b1871a5c-6f9a-48b7-86db-691023b33f51:NORMAL:127.0.0.1:45386|RBW]]} size 0
   [junit4]   2> 1988322 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:57260 is added to blk_1073741827_1003{UCState=COMMITTED, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-a1322347-01b5-4f8e-a308-0de3817fb7b3:NORMAL:127.0.0.1:45386|RBW], ReplicaUC[[DISK]DS-8c22646a-f3be-4c69-8c5a-15b7e8d62ff2:NORMAL:127.0.0.1:57260|RBW]]} size 69
   [junit4]   2> 1988322 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:57260 is added to blk_1073741828_1004{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-438a20a0-d113-4d8c-8039-c478658f4b5a:NORMAL:127.0.0.1:57260|RBW], ReplicaUC[[DISK]DS-b1871a5c-6f9a-48b7-86db-691023b33f51:NORMAL:127.0.0.1:45386|RBW]]} size 0
   [junit4]   2> 1988322 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:45386 is added to blk_1073741827_1003 size 69
   [junit4]   2> 1988456 INFO  (qtp951356604-14185) [n:127.0.0.1:58397_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard1 r:core_node5 x:MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n3] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
   [junit4]   2> 1988456 INFO  (qtp951356604-14185) [n:127.0.0.1:58397_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard1 r:core_node5 x:MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n3] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1988456 INFO  (qtp951356604-14185) [n:127.0.0.1:58397_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard1 r:core_node5 x:MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n3] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=3
   [junit4]   2> 1988473 INFO  (qtp951356604-14185) [n:127.0.0.1:58397_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard1 r:core_node5 x:MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n3] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1988473 INFO  (qtp951356604-14185) [n:127.0.0.1:58397_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard1 r:core_node5 x:MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n3] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1988479 INFO  (qtp163946297-13917) [n:127.0.0.1:58907_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard2 r:core_node8 x:MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n6] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
   [junit4]   2> 1988479 INFO  (qtp163946297-13917) [n:127.0.0.1:58907_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard2 r:core_node8 x:MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n6] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1988479 INFO  (qtp163946297-13917) [n:127.0.0.1:58907_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard2 r:core_node8 x:MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n6] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=3
   [junit4]   2> 1988490 INFO  (qtp163946297-13917) [n:127.0.0.1:58907_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard2 r:core_node8 x:MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n6] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1988490 INFO  (qtp163946297-13917) [n:127.0.0.1:58907_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard2 r:core_node8 x:MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n6] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1988526 INFO  (qtp951356604-14185) [n:127.0.0.1:58397_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard1 r:core_node5 x:MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n3] o.a.s.s.SolrIndexSearcher Opening [Searcher@69f5c07b[MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n3] main]
   [junit4]   2> 1988528 INFO  (qtp951356604-14185) [n:127.0.0.1:58397_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard1 r:core_node5 x:MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n3] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1988528 INFO  (qtp951356604-14185) [n:127.0.0.1:58397_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard1 r:core_node5 x:MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n3] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1988531 INFO  (qtp163946297-13917) [n:127.0.0.1:58907_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard2 r:core_node8 x:MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n6] o.a.s.s.SolrIndexSearcher Opening [Searcher@6cb2796e[MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n6] main]
   [junit4]   2> 1988532 INFO  (qtp163946297-13917) [n:127.0.0.1:58907_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard2 r:core_node8 x:MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n6] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1988532 INFO  (qtp163946297-13917) [n:127.0.0.1:58907_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard2 r:core_node8 x:MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n6] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1988532 INFO  (qtp163946297-13917) [n:127.0.0.1:58907_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard2 r:core_node8 x:MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n6] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 1988533 INFO  (qtp951356604-14185) [n:127.0.0.1:58397_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard1 r:core_node5 x:MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n3] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 1988534 INFO  (qtp951356604-14185) [n:127.0.0.1:58397_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard1 r:core_node5 x:MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n3] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1605829165235306496
   [junit4]   2> 1988536 INFO  (searcherExecutor-5893-thread-1-processing-n:127.0.0.1:58907_solr x:MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n6 c:MoveReplicaHDFSTest_failed_coll_false s:shard2 r:core_node8) [n:127.0.0.1:58907_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard2 r:core_node8 x:MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n6] o.a.s.c.SolrCore [MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n6] Registered new searcher Searcher@6cb2796e[MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n6] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1988536 INFO  (qtp163946297-13917) [n:127.0.0.1:58907_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard2 r:core_node8 x:MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n6] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1605829165237403648
   [junit4]   2> 1988536 INFO  (searcherExecutor-5892-thread-1-processing-n:127.0.0.1:58397_solr x:MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n3 c:MoveReplicaHDFSTest_failed_coll_false s:shard1 r:core_node5) [n:127.0.0.1:58397_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard1 r:core_node5 x:MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n3] o.a.s.c.SolrCore [MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n3] Registered new searcher Searcher@69f5c07b[MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n3] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1988542 INFO  (qtp163946297-13917) [n:127.0.0.1:58907_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard2 r:core_node8 x:MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n6] o.a.s.c.ZkShardTerms Successful update of terms at /collections/MoveReplicaHDFSTest_failed_coll_false/terms/shard2 to Terms{values={core_node8=0}, version=0}
   [junit4]   2> 1988544 INFO  (qtp163946297-13917) [n:127.0.0.1:58907_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard2 r:core_node8 x:MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n6] o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for shard shard2: total=2 found=1 timeoutin=9999ms
   [junit4]   2> 1988546 INFO  (qtp951356604-14185) [n:127.0.0.1:58397_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard1 r:core_node5 x:MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n3] o.a.s.c.ZkShardTerms Successful update of terms at /collections/MoveReplicaHDFSTest_failed_coll_false/terms/shard1 to Terms{values={core_node5=0}, version=0}
   [junit4]   2> 1988548 INFO  (qtp951356604-14185) [n:127.0.0.1:58397_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard1 r:core_node5 x:MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n3] o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for shard shard1: total=2 found=1 timeoutin=9999ms
   [junit4]   2> 1988765 INFO  (qtp722955117-13884) [n:127.0.0.1:53772_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard2 r:core_node7 x:MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n4] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
   [junit4]   2> 1988765 INFO  (qtp722955117-13884) [n:127.0.0.1:53772_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard2 r:core_node7 x:MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n4] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1988765 INFO  (qtp722955117-13884) [n:127.0.0.1:53772_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard2 r:core_node7 x:MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n4] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=3
   [junit4]   2> 1988775 INFO  (qtp722955117-13884) [n:127.0.0.1:53772_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard2 r:core_node7 x:MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n4] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1988775 INFO  (qtp722955117-13884) [n:127.0.0.1:53772_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard2 r:core_node7 x:MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n4] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1988775 INFO  (qtp902377229-13895) [n:127.0.0.1:49096_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard1 r:core_node2 x:MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
   [junit4]   2> 1988775 INFO  (qtp902377229-13895) [n:127.0.0.1:49096_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard1 r:core_node2 x:MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1988775 INFO  (qtp902377229-13895) [n:127.0.0.1:49096_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard1 r:core_node2 x:MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n1] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=3
   [junit4]   2> 1988793 INFO  (qtp722955117-13884) [n:127.0.0.1:53772_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard2 r:core_node7 x:MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n4] o.a.s.s.SolrIndexSearcher Opening [Searcher@ea63ee7[MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n4] main]
   [junit4]   2> 1988794 INFO  (qtp722955117-13884) [n:127.0.0.1:53772_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard2 r:core_node7 x:MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n4] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1988794 INFO  (qtp722955117-13884) [n:127.0.0.1:53772_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard2 r:core_node7 x:MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n4] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1988795 INFO  (qtp722955117-13884) [n:127.0.0.1:53772_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard2 r:core_node7 x:MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n4] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 1988796 INFO  (searcherExecutor-5894-thread-1-processing-n:127.0.0.1:53772_solr x:MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n4 c:MoveReplicaHDFSTest_failed_coll_false s:shard2 r:core_node7) [n:127.0.0.1:53772_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard2 r:core_node7 x:MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n4] o.a.s.c.SolrCore [MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n4] Registered new searcher Searcher@ea63ee7[MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n4] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1988796 INFO  (qtp722955117-13884) [n:127.0.0.1:53772_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard2 r:core_node7 x:MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n4] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1605829165510033408
   [junit4]   2> 1988801 INFO  (qtp902377229-13895) [n:127.0.0.1:49096_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard1 r:core_node2 x:MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1988801 INFO  (qtp902377229-13895) [n:127.0.0.1:49096_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard1 r:core_node2 x:MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1988804 INFO  (qtp722955117-13884) [n:127.0.0.1:53772_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard2 r:core_node7 x:MoveReplicaHDFSTest_failed_coll_false_shard2_replica_n4] o.a.s.c.ZkShardTerms Successful update of terms at /collections/MoveReplicaHDFSTest_failed_coll_false/terms/shard2 to Terms{values={core_node7=0, core_node8=0}, version=1}
   [junit4]   2> 1988877 INFO  (qtp902377229-13895) [n:127.0.0.1:49096_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard1 r:core_node2 x:MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@63a29112[MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n1] main]
   [junit4]   2> 1988885 INFO  (qtp902377229-13895) [n:127.0.0.1:49096_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard1 r:core_node2 x:MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1988885 INFO  (qtp902377229-13895) [n:127.0.0.1:49096_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard1 r:core_node2 x:MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1988886 INFO  (qtp902377229-13895) [n:127.0.0.1:49096_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard1 r:core_node2 x:MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 1988887 INFO  (searcherExecutor-5891-thread-1-processing-n:127.0.0.1:49096_solr x:MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n1 c:MoveReplicaHDFSTest_failed_coll_false s:shard1 r:core_node2) [n:127.0.0.1:49096_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard1 r:core_node2 x:MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n1] o.a.s.c.SolrCore [MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n1] Registered new searcher Searcher@63a29112[MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1988887 INFO  (qtp902377229-13895) [n:127.0.0.1:49096_solr c:MoveReplicaHDFSTest_failed_coll_false s:shard1 r:core_node2 x:MoveReplicaHDFSTest_failed_coll_false_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max versio

[...truncated too long message...]

ck:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.

-ivy-fail-disallowed-ivy-version:

ivy-fail:

ivy-fail:

ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/top-level-ivy-settings.xml

resolve:

jar-checksums:
    [mkdir] Created dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/null1542797446
     [copy] Copying 241 files to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/null1542797446
   [delete] Deleting directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/null1542797446

check-working-copy:
[ivy:cachepath] :: resolving dependencies :: org.eclipse.jgit#org.eclipse.jgit-caller;working
[ivy:cachepath] 	confs: [default]
[ivy:cachepath] 	found org.eclipse.jgit#org.eclipse.jgit;4.6.0.201612231935-r in public
[ivy:cachepath] 	found com.jcraft#jsch;0.1.53 in public
[ivy:cachepath] 	found com.googlecode.javaewah#JavaEWAH;1.1.6 in public
[ivy:cachepath] 	found org.apache.httpcomponents#httpclient;4.3.6 in public
[ivy:cachepath] 	found org.apache.httpcomponents#httpcore;4.3.3 in public
[ivy:cachepath] 	found commons-logging#commons-logging;1.1.3 in public
[ivy:cachepath] 	found commons-codec#commons-codec;1.6 in public
[ivy:cachepath] 	found org.slf4j#slf4j-api;1.7.2 in public
[ivy:cachepath] :: resolution report :: resolve 75ms :: artifacts dl 7ms
	---------------------------------------------------------------------
	|                  |            modules            ||   artifacts   |
	|       conf       | number| search|dwnlded|evicted|| number|dwnlded|
	---------------------------------------------------------------------
	|      default     |   8   |   0   |   0   |   0   ||   8   |   0   |
	---------------------------------------------------------------------
[wc-checker] Initializing working copy...
[wc-checker] SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
[wc-checker] SLF4J: Defaulting to no-operation (NOP) logger implementation
[wc-checker] SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
[wc-checker] Checking working copy status...

-jenkins-base:

BUILD SUCCESSFUL
Total time: 132 minutes 57 seconds
Archiving artifacts
java.lang.InterruptedException: no matches found within 10000
	at hudson.FilePath$34.hasMatch(FilePath.java:2678)
	at hudson.FilePath$34.invoke(FilePath.java:2557)
	at hudson.FilePath$34.invoke(FilePath.java:2547)
	at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2918)
Also:   hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene
		at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1741)
		at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357)
		at hudson.remoting.Channel.call(Channel.java:955)
		at hudson.FilePath.act(FilePath.java:1036)
		at hudson.FilePath.act(FilePath.java:1025)
		at hudson.FilePath.validateAntFileMask(FilePath.java:2547)
		at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
		at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
		at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
		at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
		at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
		at hudson.model.Build$BuildExecution.post2(Build.java:186)
		at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
		at hudson.model.Run.execute(Run.java:1819)
		at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
		at hudson.model.ResourceController.execute(ResourceController.java:97)
		at hudson.model.Executor.run(Executor.java:429)
Caused: hudson.FilePath$TunneledInterruptedException
	at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2920)
	at hudson.remoting.UserRequest.perform(UserRequest.java:212)
	at hudson.remoting.UserRequest.perform(UserRequest.java:54)
	at hudson.remoting.Request$2.run(Request.java:369)
	at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:748)
Caused: java.lang.InterruptedException: java.lang.InterruptedException: no matches found within 10000
	at hudson.FilePath.act(FilePath.java:1038)
	at hudson.FilePath.act(FilePath.java:1025)
	at hudson.FilePath.validateAntFileMask(FilePath.java:2547)
	at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
	at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
	at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
	at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
	at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
	at hudson.model.Build$BuildExecution.post2(Build.java:186)
	at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
	at hudson.model.Run.execute(Run.java:1819)
	at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
	at hudson.model.ResourceController.execute(ResourceController.java:97)
	at hudson.model.Executor.run(Executor.java:429)
No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/hs_err_pid*". Configuration error?
Recording test results
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)