You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2017/08/25 02:03:04 UTC

[JENKINS] Lucene-Solr-master-MacOSX (64bit/jdk1.8.0) - Build # 4144 - Still Unstable!

Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-MacOSX/4144/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseParallelGC

14 tests failed.
FAILED:  org.apache.solr.cloud.AsyncCallRequestStatusResponseTest.testAsyncCallStatusResponse

Error Message:
Expected collection 'asynccall' to have 2 shards and 1 replica null Live Nodes: [127.0.0.1:52210_solr, 127.0.0.1:52209_solr] Last available state: DocCollection(asynccall//collections/asynccall/state.json/2)={   "pullReplicas":"0",   "replicationFactor":"1",   "shards":{     "shard1":{       "range":"80000000-ffffffff",       "state":"active",       "replicas":{"core_node3":{           "core":"asynccall_shard1_replica_n1",           "base_url":"http://127.0.0.1:52210/solr",           "node_name":"127.0.0.1:52210_solr",           "state":"down",           "type":"NRT"}}},     "shard2":{       "range":"0-7fffffff",       "state":"active",       "replicas":{"core_node4":{           "core":"asynccall_shard2_replica_n2",           "base_url":"http://127.0.0.1:52209/solr",           "node_name":"127.0.0.1:52209_solr",           "state":"down",           "type":"NRT"}}}},   "router":{"name":"compositeId"},   "maxShardsPerNode":"1",   "autoAddReplicas":"false",   "nrtReplicas":"1",   "tlogReplicas":"0"}

Stack Trace:
java.lang.AssertionError: Expected collection 'asynccall' to have 2 shards and 1 replica
null
Live Nodes: [127.0.0.1:52210_solr, 127.0.0.1:52209_solr]
Last available state: DocCollection(asynccall//collections/asynccall/state.json/2)={
  "pullReplicas":"0",
  "replicationFactor":"1",
  "shards":{
    "shard1":{
      "range":"80000000-ffffffff",
      "state":"active",
      "replicas":{"core_node3":{
          "core":"asynccall_shard1_replica_n1",
          "base_url":"http://127.0.0.1:52210/solr",
          "node_name":"127.0.0.1:52210_solr",
          "state":"down",
          "type":"NRT"}}},
    "shard2":{
      "range":"0-7fffffff",
      "state":"active",
      "replicas":{"core_node4":{
          "core":"asynccall_shard2_replica_n2",
          "base_url":"http://127.0.0.1:52209/solr",
          "node_name":"127.0.0.1:52209_solr",
          "state":"down",
          "type":"NRT"}}}},
  "router":{"name":"compositeId"},
  "maxShardsPerNode":"1",
  "autoAddReplicas":"false",
  "nrtReplicas":"1",
  "tlogReplicas":"0"}
	at __randomizedtesting.SeedInfo.seed([2C7A3A69405EF0C4:767B5F856E647D1D]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.SolrCloudTestCase.waitForState(SolrCloudTestCase.java:269)
	at org.apache.solr.cloud.AsyncCallRequestStatusResponseTest.testAsyncCallStatusResponse(AsyncCallRequestStatusResponseTest.java:43)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.CustomCollectionTest.testCustomCollectionsAPI

Error Message:
Could not find collection : implicitcoll

Stack Trace:
org.apache.solr.common.SolrException: Could not find collection : implicitcoll
	at __randomizedtesting.SeedInfo.seed([2C7A3A69405EF0C4:469BB4027DC446BC]:0)
	at org.apache.solr.common.cloud.ClusterState.getCollection(ClusterState.java:109)
	at org.apache.solr.cloud.SolrCloudTestCase.getCollectionState(SolrCloudTestCase.java:247)
	at org.apache.solr.cloud.CustomCollectionTest.testCustomCollectionsAPI(CustomCollectionTest.java:68)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.CustomCollectionTest.testRouteFieldForHashRouter

Error Message:
Collection not found: routeFieldColl

Stack Trace:
org.apache.solr.common.SolrException: Collection not found: routeFieldColl
	at __randomizedtesting.SeedInfo.seed([2C7A3A69405EF0C4:844CA4B4DF3F1B9E]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.getCollectionNames(CloudSolrClient.java:1139)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:822)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
	at org.apache.solr.client.solrj.request.UpdateRequest.commit(UpdateRequest.java:233)
	at org.apache.solr.cloud.CustomCollectionTest.testRouteFieldForHashRouter(CustomCollectionTest.java:166)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.ForceLeaderTest.testReplicasInLIRNoLeader

Error Message:
Doc with id=1 not found in http://127.0.0.1:62827/zxws/u/forceleader_test_collection due to: Path not found: /id; rsp={doc=null}

Stack Trace:
java.lang.AssertionError: Doc with id=1 not found in http://127.0.0.1:62827/zxws/u/forceleader_test_collection due to: Path not found: /id; rsp={doc=null}
	at __randomizedtesting.SeedInfo.seed([2C7A3A69405EF0C4:CAED0EA979DC09A5]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.cloud.HttpPartitionTest.assertDocExists(HttpPartitionTest.java:603)
	at org.apache.solr.cloud.HttpPartitionTest.assertDocsExistInAllReplicas(HttpPartitionTest.java:556)
	at org.apache.solr.cloud.ForceLeaderTest.testReplicasInLIRNoLeader(ForceLeaderTest.java:142)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.TestTlogReplica.testDeleteById

Error Message:
Expected collection to be created with 1 shards and  2 replicas null Live Nodes: [127.0.0.1:64997_solr, 127.0.0.1:64996_solr] Last available state: null

Stack Trace:
java.lang.AssertionError: Expected collection to be created with 1 shards and  2 replicas
null
Live Nodes: [127.0.0.1:64997_solr, 127.0.0.1:64996_solr]
Last available state: null
	at __randomizedtesting.SeedInfo.seed([2C7A3A69405EF0C4:75F749FA1E0C9824]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.SolrCloudTestCase.waitForState(SolrCloudTestCase.java:269)
	at org.apache.solr.cloud.TestTlogReplica.createAndWaitForCollection(TestTlogReplica.java:716)
	at org.apache.solr.cloud.TestTlogReplica.testDeleteById(TestTlogReplica.java:603)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.TestTlogReplica.testOnlyLeaderIndexes

Error Message:


Stack Trace:
java.lang.NullPointerException
	at __randomizedtesting.SeedInfo.seed([2C7A3A69405EF0C4:307B47E435FB8E57]:0)
	at org.apache.solr.cloud.TestTlogReplica.getSolrCore(TestTlogReplica.java:846)
	at org.apache.solr.cloud.TestTlogReplica.testOnlyLeaderIndexes(TestTlogReplica.java:467)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.rest.schema.TestSchemaResource.testSchemaXmlResponse

Error Message:
Exception during query

Stack Trace:
java.lang.RuntimeException: Exception during query
	at __randomizedtesting.SeedInfo.seed([2C7A3A69405EF0C4:4D01A73432B6DD22]:0)
	at org.apache.solr.util.RestTestBase.assertQ(RestTestBase.java:149)
	at org.apache.solr.rest.schema.TestSchemaResource.testSchemaXmlResponse(TestSchemaResource.java:145)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.net.SocketException: Too many open files
	at java.net.Socket.createImpl(Socket.java:460)
	at java.net.Socket.getImpl(Socket.java:520)
	at java.net.Socket.setSoTimeout(Socket.java:1141)
	at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:119)
	at org.apache.http.impl.conn.PoolingHttpClientConnectionManager.connect(PoolingHttpClientConnectionManager.java:353)
	at org.apache.http.impl.execchain.MainClientExec.establishRoute(MainClientExec.java:380)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:236)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.solr.util.RestTestHarness.getResponse(RestTestHarness.java:215)
	at org.apache.solr.util.RestTestHarness.query(RestTestHarness.java:107)
	at org.apache.solr.util.RestTestBase.assertQ(RestTestBase.java:120)
	... 40 more


FAILED:  org.apache.solr.rest.schema.TestSchemaResource.testXMLResponse

Error Message:
Exception during query

Stack Trace:
java.lang.RuntimeException: Exception during query
	at __randomizedtesting.SeedInfo.seed([2C7A3A69405EF0C4:C396D560F5CD95A5]:0)
	at org.apache.solr.util.RestTestBase.assertQ(RestTestBase.java:149)
	at org.apache.solr.rest.schema.TestSchemaResource.testXMLResponse(TestSchemaResource.java:24)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.RuntimeException: REQUEST FAILED: xpath=count(/response/lst[@name='schema']/str[@name='name']) = 1
	xml response was: <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 404 </title>
</head>
<body>
<h2>HTTP ERROR: 404</h2>
<p>Problem accessing /solr/collection1/schema/. Reason:
<pre>    javax.servlet.UnavailableException: Error processing the request. CoreContainer is either not initialized or shutting down.</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.20.v20170531</a><hr/>
</body>
</html>

	request was:/schema/?indent=on&wt=xml
	at org.apache.solr.util.RestTestBase.assertQ(RestTestBase.java:142)
	... 40 more


FAILED:  org.apache.solr.rest.schema.TestSchemaResource.testJSONResponse

Error Message:
JSON Parse Error: char=<,position=0 AFTER='<' BEFORE='html> <head> <meta http-equiv="Content-'

Stack Trace:
org.noggit.JSONParser$ParseException: JSON Parse Error: char=<,position=0 AFTER='<' BEFORE='html> <head> <meta http-equiv="Content-'
	at __randomizedtesting.SeedInfo.seed([2C7A3A69405EF0C4:24C747D2C0C449D6]:0)
	at org.noggit.JSONParser.err(JSONParser.java:419)
	at org.noggit.JSONParser.handleNonDoubleQuoteString(JSONParser.java:775)
	at org.noggit.JSONParser.next(JSONParser.java:981)
	at org.noggit.JSONParser.nextEvent(JSONParser.java:1026)
	at org.noggit.ObjectBuilder.<init>(ObjectBuilder.java:43)
	at org.noggit.ObjectBuilder.getVal(ObjectBuilder.java:36)
	at org.noggit.ObjectBuilder.fromJSON(ObjectBuilder.java:32)
	at org.apache.solr.JSONTestUtil.match(JSONTestUtil.java:96)
	at org.apache.solr.JSONTestUtil.match(JSONTestUtil.java:73)
	at org.apache.solr.util.RestTestBase.assertJQ(RestTestBase.java:240)
	at org.apache.solr.util.RestTestBase.assertJQ(RestTestBase.java:192)
	at org.apache.solr.rest.schema.TestSchemaResource.testJSONResponse(TestSchemaResource.java:106)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.rest.schema.TestSchemaResource

Error Message:
2 threads leaked from SUITE scope at org.apache.solr.rest.schema.TestSchemaResource:     1) Thread[id=12425, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestSchemaResource]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=12424, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestSchemaResource]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.rest.schema.TestSchemaResource: 
   1) Thread[id=12425, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestSchemaResource]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=12424, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestSchemaResource]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:748)
	at __randomizedtesting.SeedInfo.seed([2C7A3A69405EF0C4]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.update.processor.TemplateUpdateProcessorTest

Error Message:
java.io.IOException: Couldn't instantiate org.apache.zookeeper.ClientCnxnSocketNIO

Stack Trace:
org.apache.solr.common.SolrException: java.io.IOException: Couldn't instantiate org.apache.zookeeper.ClientCnxnSocketNIO
	at __randomizedtesting.SeedInfo.seed([2C7A3A69405EF0C4]:0)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:171)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:117)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:112)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:99)
	at org.apache.solr.cloud.MiniSolrCloudCluster.waitForAllNodes(MiniSolrCloudCluster.java:268)
	at org.apache.solr.cloud.MiniSolrCloudCluster.<init>(MiniSolrCloudCluster.java:262)
	at org.apache.solr.cloud.SolrCloudTestCase$Builder.configure(SolrCloudTestCase.java:190)
	at org.apache.solr.update.processor.TemplateUpdateProcessorTest.setupCluster(TemplateUpdateProcessorTest.java:48)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:847)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.io.IOException: Couldn't instantiate org.apache.zookeeper.ClientCnxnSocketNIO
	at org.apache.zookeeper.ZooKeeper.getClientCnxnSocket(ZooKeeper.java:1842)
	at org.apache.zookeeper.ZooKeeper.<init>(ZooKeeper.java:447)
	at org.apache.zookeeper.ZooKeeper.<init>(ZooKeeper.java:380)
	at org.apache.solr.common.cloud.SolrZooKeeper.<init>(SolrZooKeeper.java:43)
	at org.apache.solr.common.cloud.ZkClientConnectionStrategy.createSolrZooKeeper(ZkClientConnectionStrategy.java:105)
	at org.apache.solr.common.cloud.DefaultConnectionStrategy.connect(DefaultConnectionStrategy.java:37)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:146)
	... 31 more
Caused by: java.io.IOException: Too many open files
	at sun.nio.ch.IOUtil.makePipe(Native Method)
	at sun.nio.ch.KQueueSelectorImpl.<init>(KQueueSelectorImpl.java:84)
	at sun.nio.ch.KQueueSelectorProvider.openSelector(KQueueSelectorProvider.java:42)
	at java.nio.channels.Selector.open(Selector.java:227)
	at org.apache.zookeeper.ClientCnxnSocketNIO.<init>(ClientCnxnSocketNIO.java:43)
	at sun.reflect.GeneratedConstructorAccessor147.newInstance(Unknown Source)
	at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
	at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
	at java.lang.Class.newInstance(Class.java:442)
	at org.apache.zookeeper.ZooKeeper.getClientCnxnSocket(ZooKeeper.java:1839)
	... 37 more


FAILED:  junit.framework.TestSuite.org.apache.solr.update.processor.TemplateUpdateProcessorTest

Error Message:
ObjectTracker found 21 object(s) that were not released!!! [InternalHttpClient, Overseer, ZkController, SolrZkClient, InternalHttpClient, InternalHttpClient, ZkController, InternalHttpClient, InternalHttpClient, InternalHttpClient, SolrZkClient, SolrZkClient, InternalHttpClient, InternalHttpClient, SolrZkClient, SolrZkClient, InternalHttpClient, ZkController, ZkController, InternalHttpClient, ZkController] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:289)  at org.apache.solr.handler.component.HttpShardHandlerFactory.init(HttpShardHandlerFactory.java:209)  at org.apache.solr.handler.component.ShardHandlerFactory.newInstance(ShardHandlerFactory.java:47)  at org.apache.solr.core.CoreContainer.load(CoreContainer.java:502)  at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:263)  at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:183)  at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:139)  at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873)  at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1596)  at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1659)  at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1316)  at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1145)  at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306)  at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)  at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367)  at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:384)  at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:247)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.cloud.Overseer  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.cloud.Overseer.start(Overseer.java:534)  at org.apache.solr.cloud.OverseerElectionContext.runLeaderProcess(ElectionContext.java:784)  at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170)  at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135)  at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:307)  at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:216)  at org.apache.solr.cloud.ZkController.init(ZkController.java:748)  at org.apache.solr.cloud.ZkController.<init>(ZkController.java:435)  at org.apache.solr.core.ZkContainer.initZooKeeper(ZkContainer.java:112)  at org.apache.solr.core.CoreContainer.load(CoreContainer.java:518)  at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:263)  at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:183)  at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:139)  at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873)  at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1596)  at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1659)  at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1316)  at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1145)  at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306)  at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)  at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367)  at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:384)  at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:247)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.cloud.ZkController  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.cloud.ZkController.<init>(ZkController.java:437)  at org.apache.solr.core.ZkContainer.initZooKeeper(ZkContainer.java:112)  at org.apache.solr.core.CoreContainer.load(CoreContainer.java:518)  at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:263)  at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:183)  at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:139)  at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873)  at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1596)  at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1659)  at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1316)  at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1145)  at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306)  at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)  at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367)  at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:384)  at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:247)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.common.cloud.SolrZkClient  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:185)  at org.apache.solr.cloud.ZkController.<init>(ZkController.java:311)  at org.apache.solr.core.ZkContainer.initZooKeeper(ZkContainer.java:112)  at org.apache.solr.core.CoreContainer.load(CoreContainer.java:518)  at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:263)  at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:183)  at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:139)  at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873)  at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1596)  at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1659)  at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1316)  at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1145)  at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306)  at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)  at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367)  at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:384)  at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:247)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:289)  at org.apache.solr.update.UpdateShardHandler.<init>(UpdateShardHandler.java:94)  at org.apache.solr.core.CoreContainer.load(CoreContainer.java:508)  at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:263)  at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:183)  at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:139)  at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873)  at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1596)  at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1659)  at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1316)  at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1145)  at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306)  at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)  at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367)  at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:384)  at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:247)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:289)  at org.apache.solr.handler.component.HttpShardHandlerFactory.init(HttpShardHandlerFactory.java:209)  at org.apache.solr.handler.component.ShardHandlerFactory.newInstance(ShardHandlerFactory.java:47)  at org.apache.solr.core.CoreContainer.load(CoreContainer.java:502)  at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:263)  at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:183)  at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:139)  at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873)  at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1596)  at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1659)  at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1316)  at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1145)  at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306)  at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)  at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367)  at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:384)  at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:247)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.cloud.ZkController  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.cloud.ZkController.<init>(ZkController.java:437)  at org.apache.solr.core.ZkContainer.initZooKeeper(ZkContainer.java:112)  at org.apache.solr.core.CoreContainer.load(CoreContainer.java:518)  at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:263)  at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:183)  at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:139)  at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873)  at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1596)  at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1659)  at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1316)  at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1145)  at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306)  at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)  at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367)  at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:384)  at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:247)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:289)  at org.apache.solr.handler.component.HttpShardHandlerFactory.init(HttpShardHandlerFactory.java:209)  at org.apache.solr.handler.component.ShardHandlerFactory.newInstance(ShardHandlerFactory.java:47)  at org.apache.solr.core.CoreContainer.load(CoreContainer.java:502)  at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:263)  at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:183)  at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:139)  at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873)  at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1596)  at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1659)  at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1316)  at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1145)  at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306)  at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)  at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367)  at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:384)  at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:247)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:289)  at org.apache.solr.handler.component.HttpShardHandlerFactory.init(HttpShardHandlerFactory.java:209)  at org.apache.solr.handler.component.ShardHandlerFactory.newInstance(ShardHandlerFactory.java:47)  at org.apache.solr.core.CoreContainer.load(CoreContainer.java:502)  at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:263)  at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:183)  at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:139)  at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873)  at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1596)  at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1659)  at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1316)  at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1145)  at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306)  at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)  at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367)  at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:384)  at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:247)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:289)  at org.apache.solr.update.UpdateShardHandler.<init>(UpdateShardHandler.java:94)  at org.apache.solr.core.CoreContainer.load(CoreContainer.java:508)  at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:263)  at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:183)  at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:139)  at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873)  at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1596)  at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1659)  at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1316)  at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1145)  at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306)  at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)  at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367)  at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:384)  at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:247)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.common.cloud.SolrZkClient  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:185)  at org.apache.solr.cloud.ZkController.<init>(ZkController.java:311)  at org.apache.solr.core.ZkContainer.initZooKeeper(ZkContainer.java:112)  at org.apache.solr.core.CoreContainer.load(CoreContainer.java:518)  at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:263)  at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:183)  at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:139)  at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873)  at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1596)  at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1659)  at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1316)  at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1145)  at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306)  at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)  at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367)  at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:384)  at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:247)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.common.cloud.SolrZkClient  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:185)  at org.apache.solr.cloud.ZkController.<init>(ZkController.java:311)  at org.apache.solr.core.ZkContainer.initZooKeeper(ZkContainer.java:112)  at org.apache.solr.core.CoreContainer.load(CoreContainer.java:518)  at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:263)  at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:183)  at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:139)  at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873)  at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1596)  at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1659)  at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1316)  at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1145)  at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306)  at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)  at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367)  at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:384)  at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:247)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:289)  at org.apache.solr.update.UpdateShardHandler.<init>(UpdateShardHandler.java:94)  at org.apache.solr.core.CoreContainer.load(CoreContainer.java:508)  at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:263)  at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:183)  at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:139)  at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873)  at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1596)  at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1659)  at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1316)  at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1145)  at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306)  at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)  at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367)  at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:384)  at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:247)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:289)  at org.apache.solr.update.UpdateShardHandler.<init>(UpdateShardHandler.java:94)  at org.apache.solr.core.CoreContainer.load(CoreContainer.java:508)  at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:263)  at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:183)  at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:139)  at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873)  at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1596)  at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1659)  at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1316)  at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1145)  at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306)  at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)  at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367)  at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:384)  at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:247)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.common.cloud.SolrZkClient  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:185)  at org.apache.solr.cloud.ZkController.<init>(ZkController.java:311)  at org.apache.solr.core.ZkContainer.initZooKeeper(ZkContainer.java:112)  at org.apache.solr.core.CoreContainer.load(CoreContainer.java:518)  at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:263)  at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:183)  at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:139)  at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873)  at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1596)  at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1659)  at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1316)  at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1145)  at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306)  at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)  at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367)  at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:384)  at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:247)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.common.cloud.SolrZkClient  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:185)  at org.apache.solr.cloud.ZkController.<init>(ZkController.java:311)  at org.apache.solr.core.ZkContainer.initZooKeeper(ZkContainer.java:112)  at org.apache.solr.core.CoreContainer.load(CoreContainer.java:518)  at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:263)  at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:183)  at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:139)  at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873)  at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1596)  at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1659)  at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1316)  at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1145)  at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306)  at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)  at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367)  at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:384)  at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:247)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:289)  at org.apache.solr.handler.component.HttpShardHandlerFactory.init(HttpShardHandlerFactory.java:209)  at org.apache.solr.handler.component.ShardHandlerFactory.newInstance(ShardHandlerFactory.java:47)  at org.apache.solr.core.CoreContainer.load(CoreContainer.java:502)  at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:263)  at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:183)  at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:139)  at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873)  at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1596)  at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1659)  at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1316)  at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1145)  at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHan

[...truncated too long message...]

62)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >   20) Thread[id=12459, name=jetty-launcher-3024-thread-4, state=TIMED_WAITING, group=TGRP-TemplateUpdateProcessorTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >   21) Thread[id=12468, name=qtp722824765-12468, state=RUNNABLE, group=TGRP-TemplateUpdateProcessorTest]
   [junit4]    >         at sun.nio.ch.KQueueArrayWrapper.interrupt(Native Method)
   [junit4]    >         at sun.nio.ch.KQueueArrayWrapper.interrupt(KQueueArrayWrapper.java:203)
   [junit4]    >         at sun.nio.ch.KQueueSelectorImpl.wakeup(KQueueSelectorImpl.java:257)
   [junit4]    >         at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)
   [junit4]    >         at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)
   [junit4]    >         at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:116)
   [junit4]    >         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >   22) Thread[id=12481, name=qtp2134197957-12481, state=RUNNABLE, group=TGRP-TemplateUpdateProcessorTest]
   [junit4]    >         at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
   [junit4]    >         at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)
   [junit4]    >         at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117)
   [junit4]    >         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >   23) Thread[id=12508, name=org.eclipse.jetty.server.session.HashSessionManager@5e414635Timer, state=TIMED_WAITING, group=TGRP-TemplateUpdateProcessorTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]    >         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
   [junit4]    >         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >   24) Thread[id=12457, name=jetty-launcher-3024-thread-2, state=TIMED_WAITING, group=TGRP-TemplateUpdateProcessorTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >   25) Thread[id=12497, name=org.eclipse.jetty.server.session.HashSessionManager@181c1b68Timer, state=TIMED_WAITING, group=TGRP-TemplateUpdateProcessorTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]    >         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
   [junit4]    >         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >   26) Thread[id=12552, name=jetty-launcher-3024-thread-5-SendThread(127.0.0.1:52491), state=TIMED_WAITING, group=TGRP-TemplateUpdateProcessorTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)
   [junit4]    >   27) Thread[id=12559, name=zkCallback-3049-thread-1, state=TIMED_WAITING, group=TGRP-TemplateUpdateProcessorTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([2C7A3A69405EF0C4]:0)
   [junit4] Completed [305/732 (6!)] on J0 in 46.10s, 0 tests, 1 failure, 3 errors <<< FAILURES!

[...truncated 44616 lines...]

[JENKINS] Lucene-Solr-master-MacOSX (64bit/jdk1.8.0) - Build # 4160 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-MacOSX/4160/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseConcMarkSweepGC

2 tests failed.
FAILED:  org.apache.solr.cloud.ShardSplitTest.testSplitAfterFailedSplit

Error Message:
expected:<1> but was:<2>

Stack Trace:
java.lang.AssertionError: expected:<1> but was:<2>
	at __randomizedtesting.SeedInfo.seed([BE0EB204AEA5718D:474321AB92D03C07]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.junit.Assert.assertEquals(Assert.java:456)
	at org.apache.solr.cloud.ShardSplitTest.testSplitAfterFailedSplit(ShardSplitTest.java:279)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.TestPullReplica.testKillPullReplica

Error Message:
Didn't update state null Live Nodes: [127.0.0.1:49954_solr, 127.0.0.1:49953_solr] Last available state: DocCollection(pull_replica_test_kill_pull_replica//collections/pull_replica_test_kill_pull_replica/state.json/6)={   "pullReplicas":"1",   "replicationFactor":"1",   "shards":{"shard1":{       "range":"80000000-7fffffff",       "state":"active",       "replicas":{         "core_node3":{           "core":"pull_replica_test_kill_pull_replica_shard1_replica_n1",           "base_url":"http://127.0.0.1:49953/solr",           "node_name":"127.0.0.1:49953_solr",           "state":"active",           "type":"NRT",           "leader":"true"},         "core_node4":{           "core":"pull_replica_test_kill_pull_replica_shard1_replica_p2",           "base_url":"http://127.0.0.1:49954/solr",           "node_name":"127.0.0.1:49954_solr",           "state":"down",           "type":"PULL"}}}},   "router":{"name":"compositeId"},   "maxShardsPerNode":"100",   "autoAddReplicas":"false",   "nrtReplicas":"1",   "tlogReplicas":"0"}

Stack Trace:
java.lang.AssertionError: Didn't update state
null
Live Nodes: [127.0.0.1:49954_solr, 127.0.0.1:49953_solr]
Last available state: DocCollection(pull_replica_test_kill_pull_replica//collections/pull_replica_test_kill_pull_replica/state.json/6)={
  "pullReplicas":"1",
  "replicationFactor":"1",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node3":{
          "core":"pull_replica_test_kill_pull_replica_shard1_replica_n1",
          "base_url":"http://127.0.0.1:49953/solr",
          "node_name":"127.0.0.1:49953_solr",
          "state":"active",
          "type":"NRT",
          "leader":"true"},
        "core_node4":{
          "core":"pull_replica_test_kill_pull_replica_shard1_replica_p2",
          "base_url":"http://127.0.0.1:49954/solr",
          "node_name":"127.0.0.1:49954_solr",
          "state":"down",
          "type":"PULL"}}}},
  "router":{"name":"compositeId"},
  "maxShardsPerNode":"100",
  "autoAddReplicas":"false",
  "nrtReplicas":"1",
  "tlogReplicas":"0"}
	at __randomizedtesting.SeedInfo.seed([BE0EB204AEA5718D:32FFAE910E1490B5]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.SolrCloudTestCase.waitForState(SolrCloudTestCase.java:269)
	at org.apache.solr.cloud.TestPullReplica.testKillPullReplica(TestPullReplica.java:480)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 11497 lines...]
   [junit4] Suite: org.apache.solr.cloud.ShardSplitTest
   [junit4]   2> 490930 INFO  (SUITE-ShardSplitTest-seed#[BE0EB204AEA5718D]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_BE0EB204AEA5718D-001/init-core-data-001
   [junit4]   2> 490931 WARN  (SUITE-ShardSplitTest-seed#[BE0EB204AEA5718D]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=5 numCloses=5
   [junit4]   2> 490933 INFO  (SUITE-ShardSplitTest-seed#[BE0EB204AEA5718D]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 490939 INFO  (SUITE-ShardSplitTest-seed#[BE0EB204AEA5718D]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776) w/ MAC_OS_X supressed clientAuth
   [junit4]   2> 490939 INFO  (SUITE-ShardSplitTest-seed#[BE0EB204AEA5718D]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /alaw/
   [junit4]   2> 490950 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 490951 INFO  (Thread-756) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 490951 INFO  (Thread-756) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 490954 ERROR (Thread-756) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 491053 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.a.s.c.ZkTestServer start zk server on port:56041
   [junit4]   2> 491104 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 491112 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
   [junit4]   2> 491114 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 491118 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 491120 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 491124 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 491125 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 491127 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 491131 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 491133 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 491134 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 491139 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.a.s.c.AbstractFullDistribZkTestBase Will use TLOG replicas unless explicitly asked otherwise
   [junit4]   2> 491463 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 491464 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7ac8e299{/alaw,null,AVAILABLE}
   [junit4]   2> 491465 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@601897d4{HTTP/1.1,[http/1.1]}{127.0.0.1:56044}
   [junit4]   2> 491467 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.e.j.s.Server Started @494985ms
   [junit4]   2> 491467 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_BE0EB204AEA5718D-001/tempDir-001/control/data, hostContext=/alaw, hostPort=56044, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_BE0EB204AEA5718D-001/control-001/cores}
   [junit4]   2> 491467 ERROR (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 491467 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solrâ„¢ version 8.0.0
   [junit4]   2> 491467 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 491467 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 491467 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-09-01T00:52:27.676Z
   [junit4]   2> 491470 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 491470 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_BE0EB204AEA5718D-001/control-001/solr.xml
   [junit4]   2> 491476 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@367d25d9, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 491479 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:56041/solr
   [junit4]   2> 491624 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [n:127.0.0.1:56044_alaw    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 491625 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [n:127.0.0.1:56044_alaw    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:56044_alaw
   [junit4]   2> 491626 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [n:127.0.0.1:56044_alaw    ] o.a.s.c.Overseer Overseer (id=98581030316408836-127.0.0.1:56044_alaw-n_0000000000) starting
   [junit4]   2> 491642 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [n:127.0.0.1:56044_alaw    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:56044_alaw
   [junit4]   2> 491644 INFO  (zkCallback-544-thread-1-processing-n:127.0.0.1:56044_alaw) [n:127.0.0.1:56044_alaw    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 491742 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [n:127.0.0.1:56044_alaw    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@367d25d9
   [junit4]   2> 491749 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [n:127.0.0.1:56044_alaw    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@367d25d9
   [junit4]   2> 491749 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [n:127.0.0.1:56044_alaw    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@367d25d9
   [junit4]   2> 491751 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [n:127.0.0.1:56044_alaw    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_BE0EB204AEA5718D-001/control-001/cores
   [junit4]   2> 491785 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 491786 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:56041/solr ready
   [junit4]   2> 491793 INFO  (qtp1459346964-4206) [n:127.0.0.1:56044_alaw    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:56044_alaw&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 491797 INFO  (OverseerThreadFactory-1323-thread-1-processing-n:127.0.0.1:56044_alaw) [n:127.0.0.1:56044_alaw    ] o.a.s.c.CreateCollectionCmd Create collection control_collection
   [junit4]   2> 491908 INFO  (qtp1459346964-4208) [n:127.0.0.1:56044_alaw    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 491909 INFO  (qtp1459346964-4208) [n:127.0.0.1:56044_alaw    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 492018 INFO  (zkCallback-544-thread-1-processing-n:127.0.0.1:56044_alaw) [n:127.0.0.1:56044_alaw    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 492931 INFO  (qtp1459346964-4208) [n:127.0.0.1:56044_alaw c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 492945 INFO  (qtp1459346964-4208) [n:127.0.0.1:56044_alaw c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
   [junit4]   2> 493040 INFO  (qtp1459346964-4208) [n:127.0.0.1:56044_alaw c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 493062 INFO  (qtp1459346964-4208) [n:127.0.0.1:56044_alaw c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
   [junit4]   2> 493063 INFO  (qtp1459346964-4208) [n:127.0.0.1:56044_alaw c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@367d25d9
   [junit4]   2> 493063 INFO  (qtp1459346964-4208) [n:127.0.0.1:56044_alaw c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 493063 INFO  (qtp1459346964-4208) [n:127.0.0.1:56044_alaw c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_BE0EB204AEA5718D-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_BE0EB204AEA5718D-001/control-001/cores/control_collection_shard1_replica_n1/data/]
   [junit4]   2> 493067 INFO  (qtp1459346964-4208) [n:127.0.0.1:56044_alaw c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=14, maxMergeAtOnceExplicit=40, maxMergedSegmentMB=26.4208984375, floorSegmentMB=0.791015625, forceMergeDeletesPctAllowed=17.283805331750646, segmentsPerTier=29.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 493080 WARN  (qtp1459346964-4208) [n:127.0.0.1:56044_alaw c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 493137 INFO  (qtp1459346964-4208) [n:127.0.0.1:56044_alaw c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 493137 INFO  (qtp1459346964-4208) [n:127.0.0.1:56044_alaw c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 493137 INFO  (qtp1459346964-4208) [n:127.0.0.1:56044_alaw c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 493137 INFO  (qtp1459346964-4208) [n:127.0.0.1:56044_alaw c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 493138 INFO  (qtp1459346964-4208) [n:127.0.0.1:56044_alaw c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=32, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 493139 INFO  (qtp1459346964-4208) [n:127.0.0.1:56044_alaw c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@2635e1ae[control_collection_shard1_replica_n1] main]
   [junit4]   2> 493140 INFO  (qtp1459346964-4208) [n:127.0.0.1:56044_alaw c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 493141 INFO  (qtp1459346964-4208) [n:127.0.0.1:56044_alaw c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 493141 INFO  (qtp1459346964-4208) [n:127.0.0.1:56044_alaw c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 493142 INFO  (qtp1459346964-4208) [n:127.0.0.1:56044_alaw c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1577296487357874176
   [junit4]   2> 493156 INFO  (searcherExecutor-1326-thread-1-processing-n:127.0.0.1:56044_alaw x:control_collection_shard1_replica_n1 s:shard1 c:control_collection) [n:127.0.0.1:56044_alaw c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@2635e1ae[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 493163 INFO  (qtp1459346964-4208) [n:127.0.0.1:56044_alaw c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 493163 INFO  (qtp1459346964-4208) [n:127.0.0.1:56044_alaw c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 493163 INFO  (qtp1459346964-4208) [n:127.0.0.1:56044_alaw c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:56044/alaw/control_collection_shard1_replica_n1/
   [junit4]   2> 493163 INFO  (qtp1459346964-4208) [n:127.0.0.1:56044_alaw c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 493163 INFO  (qtp1459346964-4208) [n:127.0.0.1:56044_alaw c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:56044/alaw/control_collection_shard1_replica_n1/ has no replicas
   [junit4]   2> 493163 INFO  (qtp1459346964-4208) [n:127.0.0.1:56044_alaw c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 493172 INFO  (qtp1459346964-4208) [n:127.0.0.1:56044_alaw c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:56044/alaw/control_collection_shard1_replica_n1/ shard1
   [junit4]   2> 493278 INFO  (zkCallback-544-thread-1-processing-n:127.0.0.1:56044_alaw) [n:127.0.0.1:56044_alaw    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 493286 INFO  (qtp1459346964-4208) [n:127.0.0.1:56044_alaw c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 493287 INFO  (qtp1459346964-4208) [n:127.0.0.1:56044_alaw c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1379
   [junit4]   2> 493291 INFO  (qtp1459346964-4206) [n:127.0.0.1:56044_alaw    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 493398 INFO  (zkCallback-544-thread-1-processing-n:127.0.0.1:56044_alaw) [n:127.0.0.1:56044_alaw    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 493800 INFO  (OverseerCollectionConfigSetProcessor-98581030316408836-127.0.0.1:56044_alaw-n_0000000000) [n:127.0.0.1:56044_alaw    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 494292 INFO  (qtp1459346964-4206) [n:127.0.0.1:56044_alaw    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:56044_alaw&wt=javabin&version=2} status=0 QTime=2499
   [junit4]   2> 494298 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 494299 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:56041/solr ready
   [junit4]   2> 494299 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 494300 INFO  (qtp1459346964-4204) [n:127.0.0.1:56044_alaw    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=2&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 494303 INFO  (OverseerThreadFactory-1323-thread-2-processing-n:127.0.0.1:56044_alaw) [n:127.0.0.1:56044_alaw    ] o.a.s.c.CreateCollectionCmd Create collection collection1
   [junit4]   2> 494304 WARN  (OverseerThreadFactory-1323-thread-2-processing-n:127.0.0.1:56044_alaw) [n:127.0.0.1:56044_alaw    ] o.a.s.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
   [junit4]   2> 494521 INFO  (qtp1459346964-4204) [n:127.0.0.1:56044_alaw    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 494521 INFO  (qtp1459346964-4204) [n:127.0.0.1:56044_alaw    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=2&wt=javabin&version=2} status=0 QTime=220
   [junit4]   2> 494779 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_BE0EB204AEA5718D-001/shard-1-001 of type TLOG
   [junit4]   2> 494779 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 494781 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1f7b48f0{/alaw,null,AVAILABLE}
   [junit4]   2> 494781 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@366cbb89{HTTP/1.1,[http/1.1]}{127.0.0.1:56053}
   [junit4]   2> 494783 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.e.j.s.Server Started @498302ms
   [junit4]   2> 494783 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_BE0EB204AEA5718D-001/tempDir-001/jetty1, replicaType=TLOG, solrconfig=solrconfig.xml, hostContext=/alaw, hostPort=56053, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_BE0EB204AEA5718D-001/shard-1-001/cores}
   [junit4]   2> 494783 ERROR (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 494784 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solrâ„¢ version 8.0.0
   [junit4]   2> 494784 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 494784 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 494784 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-09-01T00:52:30.993Z
   [junit4]   2> 494786 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 494787 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_BE0EB204AEA5718D-001/shard-1-001/solr.xml
   [junit4]   2> 494799 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@367d25d9, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 494802 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:56041/solr
   [junit4]   2> 494814 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [n:127.0.0.1:56053_alaw    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 494817 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [n:127.0.0.1:56053_alaw    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 494820 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [n:127.0.0.1:56053_alaw    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:56053_alaw
   [junit4]   2> 494822 INFO  (zkCallback-544-thread-1-processing-n:127.0.0.1:56044_alaw) [n:127.0.0.1:56044_alaw    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 494824 INFO  (zkCallback-551-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 494824 INFO  (zkCallback-556-thread-1-processing-n:127.0.0.1:56053_alaw) [n:127.0.0.1:56053_alaw    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 494923 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [n:127.0.0.1:56053_alaw    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@367d25d9
   [junit4]   2> 494933 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [n:127.0.0.1:56053_alaw    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@367d25d9
   [junit4]   2> 494933 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [n:127.0.0.1:56053_alaw    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@367d25d9
   [junit4]   2> 494935 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [n:127.0.0.1:56053_alaw    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_BE0EB204AEA5718D-001/shard-1-001/cores
   [junit4]   2> 494974 INFO  (qtp1459346964-4207) [n:127.0.0.1:56044_alaw    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:56053_alaw&action=ADDREPLICA&collection=collection1&shard=shard2&type=TLOG&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 494978 INFO  (OverseerCollectionConfigSetProcessor-98581030316408836-127.0.0.1:56044_alaw-n_0000000000) [n:127.0.0.1:56044_alaw    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 494978 INFO  (OverseerThreadFactory-1323-thread-3-processing-n:127.0.0.1:56044_alaw) [n:127.0.0.1:56044_alaw    ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:56053_alaw for creating new replica
   [junit4]   2> 495094 INFO  (qtp1503893309-4247) [n:127.0.0.1:56053_alaw    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_t41&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG
   [junit4]   2> 495094 INFO  (qtp1503893309-4247) [n:127.0.0.1:56053_alaw    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 495203 INFO  (zkCallback-556-thread-1-processing-n:127.0.0.1:56053_alaw) [n:127.0.0.1:56053_alaw    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 496123 INFO  (qtp1503893309-4247) [n:127.0.0.1:56053_alaw c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 496136 INFO  (qtp1503893309-4247) [n:127.0.0.1:56053_alaw c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.s.IndexSchema [collection1_shard2_replica_t41] Schema name=test
   [junit4]   2> 496233 INFO  (qtp1503893309-4247) [n:127.0.0.1:56053_alaw c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 496254 INFO  (qtp1503893309-4247) [n:127.0.0.1:56053_alaw c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_t41' using configuration from collection collection1, trusted=true
   [junit4]   2> 496255 INFO  (qtp1503893309-4247) [n:127.0.0.1:56053_alaw c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_t41' (registry 'solr.core.collection1.shard2.replica_t41') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@367d25d9
   [junit4]   2> 496255 INFO  (qtp1503893309-4247) [n:127.0.0.1:56053_alaw c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 496255 INFO  (qtp1503893309-4247) [n:127.0.0.1:56053_alaw c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.SolrCore [[collection1_shard2_replica_t41] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_BE0EB204AEA5718D-001/shard-1-001/cores/collection1_shard2_replica_t41], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_BE0EB204AEA5718D-001/shard-1-001/cores/collection1_shard2_replica_t41/data/]
   [junit4]   2> 496258 INFO  (qtp1503893309-4247) [n:127.0.0.1:56053_alaw c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=14, maxMergeAtOnceExplicit=40, maxMergedSegmentMB=26.4208984375, floorSegmentMB=0.791015625, forceMergeDeletesPctAllowed=17.283805331750646, segmentsPerTier=29.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 496266 WARN  (qtp1503893309-4247) [n:127.0.0.1:56053_alaw c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 496315 INFO  (qtp1503893309-4247) [n:127.0.0.1:56053_alaw c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 496315 INFO  (qtp1503893309-4247) [n:127.0.0.1:56053_alaw c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 496315 INFO  (qtp1503893309-4247) [n:127.0.0.1:56053_alaw c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 496315 INFO  (qtp1503893309-4247) [n:127.0.0.1:56053_alaw c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 496316 INFO  (qtp1503893309-4247) [n:127.0.0.1:56053_alaw c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=32, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 496317 INFO  (qtp1503893309-4247) [n:127.0.0.1:56053_alaw c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.s.SolrIndexSearcher Opening [Searcher@41f8dd34[collection1_shard2_replica_t41] main]
   [junit4]   2> 496318 INFO  (qtp1503893309-4247) [n:127.0.0.1:56053_alaw c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 496319 INFO  (qtp1503893309-4247) [n:127.0.0.1:56053_alaw c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 496319 INFO  (qtp1503893309-4247) [n:127.0.0.1:56053_alaw c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 496319 INFO  (qtp1503893309-4247) [n:127.0.0.1:56053_alaw c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1577296490689200128
   [junit4]   2> 496322 INFO  (searcherExecutor-1337-thread-1-processing-n:127.0.0.1:56053_alaw x:collection1_shard2_replica_t41 s:shard2 c:collection1) [n:127.0.0.1:56053_alaw c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.SolrCore [collection1_shard2_replica_t41] Registered new searcher Searcher@41f8dd34[collection1_shard2_replica_t41] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 496328 INFO  (qtp1503893309-4247) [n:127.0.0.1:56053_alaw c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 496328 INFO  (qtp1503893309-4247) [n:127.0.0.1:56053_alaw c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 496328 INFO  (qtp1503893309-4247) [n:127.0.0.1:56053_alaw c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:56053/alaw/collection1_shard2_replica_t41/
   [junit4]   2> 496328 INFO  (qtp1503893309-4247) [n:127.0.0.1:56053_alaw c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 496345 INFO  (qtp1503893309-4247) [n:127.0.0.1:56053_alaw c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.SyncStrategy http://127.0.0.1:56053/alaw/collection1_shard2_replica_t41/ has no replicas
   [junit4]   2> 496345 INFO  (qtp1503893309-4247) [n:127.0.0.1:56053_alaw c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 496346 INFO  (qtp1503893309-4247) [n:127.0.0.1:56053_alaw c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.ZkController collection1_shard2_replica_t41 stopping background replication from leader
   [junit4]   2> 496350 INFO  (qtp1503893309-4247) [n:127.0.0.1:56053_alaw c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:56053/alaw/collection1_shard2_replica_t41/ shard2
   [junit4]   2> 496458 INFO  (zkCallback-556-thread-1-processing-n:127.0.0.1:56053_alaw) [n:127.0.0.1:56053_alaw    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 496502 INFO  (qtp1503893309-4247) [n:127.0.0.1:56053_alaw c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 496504 INFO  (qtp1503893309-4247) [n:127.0.0.1:56053_alaw c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_t41&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=1409
   [junit4]   2> 496507 INFO  (qtp1459346964-4207) [n:127.0.0.1:56044_alaw    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:56053_alaw&action=ADDREPLICA&collection=collection1&shard=shard2&type=TLOG&wt=javabin&version=2} status=0 QTime=1532
   [junit4]   2> 496608 INFO  (zkCallback-556-thread-1-processing-n:127.0.0.1:56053_alaw) [n:127.0.0.1:56053_alaw    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 496810 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_BE0EB204AEA5718D-001/shard-2-001 of type TLOG
   [junit4]   2> 496810 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 496811 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3aaffaab{/alaw,null,AVAILABLE}
   [junit4]   2> 496812 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@2d9880cc{HTTP/1.1,[http/1.1]}{127.0.0.1:56067}
   [junit4]   2> 496812 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.e.j.s.Server Started @500330ms
   [junit4]   2> 496814 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_BE0EB204AEA5718D-001/tempDir-001/jetty2, replicaType=TLOG, solrconfig=solrconfig.xml, hostContext=/alaw, hostPort=56067, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_BE0EB204AEA5718D-001/shard-2-001/cores}
   [junit4]   2> 496814 ERROR (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 496816 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solrâ„¢ version 8.0.0
   [junit4]   2> 496816 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 496816 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 496816 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-09-01T00:52:33.025Z
   [junit4]   2> 496819 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 496819 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_BE0EB204AEA5718D-001/shard-2-001/solr.xml
   [junit4]   2> 496832 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@367d25d9, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 496835 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:56041/solr
   [junit4]   2> 496847 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [n:127.0.0.1:56067_alaw    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 496851 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [n:127.0.0.1:56067_alaw    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 496853 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [n:127.0.0.1:56067_alaw    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:56067_alaw
   [junit4]   2> 496855 INFO  (zkCallback-544-thread-1-processing-n:127.0.0.1:56044_alaw) [n:127.0.0.1:56044_alaw    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 496857 INFO  (zkCallback-562-thread-1-processing-n:127.0.0.1:56067_alaw) [n:127.0.0.1:56067_alaw    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 496855 INFO  (zkCallback-556-thread-1-processing-n:127.0.0.1:56053_alaw) [n:127.0.0.1:56053_alaw    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 496855 INFO  (zkCallback-551-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 496936 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [n:127.0.0.1:56067_alaw    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@367d25d9
   [junit4]   2> 496948 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [n:127.0.0.1:56067_alaw    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@367d25d9
   [junit4]   2> 496948 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [n:127.0.0.1:56067_alaw    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@367d25d9
   [junit4]   2> 496949 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [n:127.0.0.1:56067_alaw    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_BE0EB204AEA5718D-001/shard-2-001/cores
   [junit4]   2> 496983 INFO  (OverseerCollectionConfigSetProcessor-98581030316408836-127.0.0.1:56044_alaw-n_0000000000) [n:127.0.0.1:56044_alaw    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 496984 INFO  (qtp1459346964-4208) [n:127.0.0.1:56044_alaw    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:56067_alaw&action=ADDREPLICA&collection=collection1&shard=shard1&type=TLOG&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 496988 INFO  (OverseerThreadFactory-1323-thread-4-processing-n:127.0.0.1:56044_alaw) [n:127.0.0.1:56044_alaw    ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:56067_alaw for creating new replica
   [junit4]   2> 496997 INFO  (qtp526022085-4277) [n:127.0.0.1:56067_alaw    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_t43&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG
   [junit4]   2> 496997 INFO  (qtp526022085-4277) [n:127.0.0.1:56067_alaw    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 497107 INFO  (zkCallback-562-thread-1-processing-n:127.0.0.1:56067_alaw) [n:127.0.0.1:56067_alaw    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 497107 INFO  (zkCallback-556-thread-1-processing-n:127.0.0.1:56053_alaw) [n:127.0.0.1:56053_alaw    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 498028 INFO  (qtp526022085-4277) [n:127.0.0.1:56067_alaw c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 498044 INFO  (qtp526022085-4277) [n:127.0.0.1:56067_alaw c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.s.IndexSchema [collection1_shard1_replica_t43] Schema name=test
   [junit4]   2> 498148 INFO  (qtp526022085-4277) [n:127.0.0.1:56067_alaw c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 498174 INFO  (qtp526022085-4277) [n:127.0.0.1:56067_alaw c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_t43' using configuration from collection collection1, trusted=true
   [junit4]   2> 498174 INFO  (qtp526022085-4277) [n:127.0.0.1:56067_alaw c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.replica_t43' (registry 'solr.core.collection1.shard1.replica_t43') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@367d25d9
   [junit4]   2> 498174 INFO  (qtp526022085-4277) [n:127.0.0.1:56067_alaw c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 498175 INFO  (qtp526022085-4277) [n:127.0.0.1:56067_alaw c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.c.SolrCore [[collection1_shard1_replica_t43] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_BE0EB204AEA5718D-001/shard-2-001/cores/collection1_shard1_replica_t43], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_BE0EB204AEA5718D-001/shard-2-001/cores/collection1_shard1_replica_t43/data/]
   [junit4]   2> 498179 INFO  (qtp526022085-4277) [n:127.0.0.1:56067_alaw c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=14, maxMergeAtOnceExplicit=40, maxMergedSegmentMB=26.4208984375, floorSegmentMB=0.791015625, forceMergeDeletesPctAllowed=17.283805331750646, segmentsPerTier=29.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 498183 WARN  (qtp526022085-4277) [n:127.0.0.1:56067_alaw c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 498238 INFO  (qtp526022085-4277) [n:127.0.0.1:56067_alaw c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 498238 INFO  (qtp526022085-4277) [n:127.0.0.1:56067_alaw c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 498247 INFO  (qtp526022085-4277) [n:127.0.0.1:56067_alaw c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 498247 INFO  (qtp526022085-4277) [n:127.0.0.1:56067_alaw c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 498248 INFO  (qtp526022085-4277) [n:127.0.0.1:56067_alaw c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=32, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 498249 INFO  (qtp526022085-4277) [n:127.0.0.1:56067_alaw c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.s.SolrIndexSearcher Opening [Searcher@4cc9b4c[collection1_shard1_replica_t43] main]
   [junit4]   2> 498250 INFO  (qtp526022085-4277) [n:127.0.0.1:56067_alaw c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 498251 INFO  (qtp526022085-4277) [n:127.0.0.1:56067_alaw c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 498251 INFO  (qtp526022085-4277) [n:127.0.0.1:56067_alaw c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 498251 INFO  (qtp526022085-4277) [n:127.0.0.1:56067_alaw c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1577296492715048960
   [junit4]   2> 498254 INFO  (searcherExecutor-1348-thread-1-processing-n:127.0.0.1:56067_alaw x:collection1_shard1_replica_t43 s:shard1 c:collection1) [n:127.0.0.1:56067_alaw c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.c.SolrCore [collection1_shard1_replica_t43] Registered new searcher Searcher@4cc9b4c[collection1_shard1_replica_t43] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 498260 INFO  (qtp526022085-4277) [n:127.0.0.1:56067_alaw c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 498260 INFO  (qtp526022085-4277) [n:127.0.0.1:56067_alaw c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 498260 INFO  (qtp526022085-4277) [n:127.0.0.1:56067_alaw c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:56067/alaw/collection1_shard1_replica_t43/
   [junit4]   2> 498260 INFO  (qtp526022085-4277) [n:127.0.0.1:56067_alaw c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 498260 INFO  (qtp526022085-4277) [n:127.0.0.1:56067_alaw c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.c.SyncStrategy http://127.0.0.1:56067/alaw/collection1_shard1_replica_t43/ has no replicas
   [junit4]   2> 498260 INFO  (qtp526022085-4277) [n:127.0.0.1:56067_alaw c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 498261 INFO  (qtp526022085-4277) [n:127.0.0.1:56067_alaw c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.c.ZkController collection1_shard1_replica_t43 stopping background replication from leader
   [junit4]   2> 498265 INFO  (qtp526022085-4277) [n:127.0.0.1:56067_alaw c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:56067/alaw/collection1_shard1_replica_t43/ shard1
   [junit4]   2> 498370 INFO  (zkCallback-562-thread-1-processing-n:127.0.0.1:56067_alaw) [n:127.0.0.1:56067_alaw    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 498371 INFO  (zkCallback-556-thread-1-processing-n:127.0.0.1:56053_alaw) [n:127.0.0.1:56053_alaw    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 498379 INFO  (qtp526022085-4277) [n:127.0.0.1:56067_alaw c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 498384 INFO  (qtp526022085-4277) [n:127.0.0.1:56067_alaw c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_t43&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=1387
   [junit4]   2> 498391 INFO  (qtp1459346964-4208) [n:127.0.0.1:56044_alaw    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:56067_alaw&action=ADDREPLICA&collection=collection1&shard=shard1&type=TLOG&wt=javabin&version=2} status=0 QTime=1406
   [junit4]   2> 498497 INFO  (zkCallback-562-thread-1-processing-n:127.0.0.1:56067_alaw) [n:127.0.0.1:56067_alaw    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 498497 INFO  (zkCallback-556-thread-1-processing-n:127.0.0.1:56053_alaw) [n:127.0.0.1:56053_alaw    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
   [junit4]   2> 498629 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_BE0EB204AEA5718D-001/shard-3-001 of type TLOG
   [junit4]   2> 498630 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 498631 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4d5a966b{/alaw,null,AVAILABLE}
   [junit4]   2> 498631 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@36513553{HTTP/1.1,[http/1.1]}{127.0.0.1:56073}
   [junit4]   2> 498631 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.e.j.s.Server Started @502150ms
   [junit4]   2> 498631 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_BE0EB204AEA5718D-001/tempDir-001/jetty3, replicaType=TLOG, solrconfig=solrconfig.xml, hostContext=/alaw, hostPort=56073, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_BE0EB204AEA5718D-001/shard-3-001/cores}
   [junit4]   2> 498633 ERROR (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 498635 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solrâ„¢ version 8.0.0
   [junit4]   2> 498635 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 498635 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 498635 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-09-01T00:52:34.844Z
   [junit4]   2> 498638 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 498638 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_BE0EB204AEA5718D-001/shard-3-001/solr.xml
   [junit4]   2> 498649 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@367d25d9, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 498652 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:56041/solr
   [junit4]   2> 498665 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [n:127.0.0.1:56073_alaw    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
   [junit4]   2> 498669 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [n:127.0.0.1:56073_alaw    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 498671 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [n:127.0.0.1:56073_alaw    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:56073_alaw
   [junit4]   2> 498674 INFO  (zkCallback-562-thread-1-processing-n:127.0.0.1:56067_alaw) [n:127.0.0.1:56067_alaw    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 498676 INFO  (zkCallback-551-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 498678 INFO  (zkCallback-568-thread-1-processing-n:127.0.0.1:56073_alaw) [n:127.0.0.1:56073_alaw    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 498678 INFO  (zkCallback-544-thread-1-processing-n:127.0.0.1:56044_alaw) [n:127.0.0.1:56044_alaw    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 498678 INFO  (zkCallback-556-thread-1-processing-n:127.0.0.1:56053_alaw) [n:127.0.0.1:56053_alaw    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 498748 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [n:127.0.0.1:56073_alaw    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@367d25d9
   [junit4]   2> 498755 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [n:127.0.0.1:56073_alaw    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@367d25d9
   [junit4]   2> 498755 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [n:127.0.0.1:56073_alaw    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@367d25d9
   [junit4]   2> 498757 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[BE0EB204AEA5718D]) [n:127.0.0.1:56073_alaw    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_BE0EB204AEA5718D-001/shard-3-001/cores
   [junit4]   2> 498802 INFO  (qtp1459346964-4206) [n:127.0.0.1:56044_alaw    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:56073_alaw&action=ADDREPLICA&collection=collection1&shard=shard2&type=TLOG&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 498805 INFO  (OverseerCollectionConfigSetProcessor-98581030316408836-127.0.0.1:56044_alaw-n_0000000000) [n:127.0.0.1:56044_alaw    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000006 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 498806 INFO  (OverseerThreadFactory-1323-thread-5-processing-n:127.0.0.1:56044_alaw) [n:127.0.0.1:56044_alaw    ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:56073_alaw for creating new replica
   [junit4]   2> 498815 INFO  (qtp1064062937-4306) [n:127.0.0.1:56073_alaw    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_t45&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG
   [junit4]   2> 498815 INFO  (qtp1064062937-4306) [n:127.0.0.1:56073_alaw    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 498936 INFO  (zkCallback-568-thread-1-processing-n:127.0.0.1:56073_alaw) [n:127.0.0.1:56073_alaw    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 498936 INFO  (zkCallback-562-thread-1-processing-n:127.0.0.1:56067_alaw) [n:127.0.0.1:56067_alaw    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 498936 INFO  (zkCallback-556-thread-1-processing-n:127.0.0.1:56053_alaw) [n:127.0.0.1:56053_alaw    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 499840 INFO  (qtp1064062937-4306) [n:127.0.0.1:56073_alaw c:collection1 s:shard2  x:collection1_shard2_replica_t45] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 499853 INFO  (qtp1064062937-4306) [n:127.0.0.1:56073_alaw c:collection1 s:shard2  x:collection1_shard2_replica_t45] o.a.s.s.IndexSchema [collection1_shard2_replica_t45] Schema name=test
   [junit4]   2> 499938 INFO  (qtp1064062937-4306) [n:127.0.0.1:56073_alaw c:collection1 s:shard2  x:collection1_shard2_replica_t45] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 499960 INFO  (qtp1064062937-4306) [n:127.0.0.1:56073_alaw c:collection1 s:shard2  x:collection1_shard2_replica_t45] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_t45' using configuration from collection collection1, trusted=true
   [junit4]   2> 499960 INFO  (qtp1064062937-4306) [n:127.0.0.1:56073_alaw c:collection1 s:shard2  x:collection1_shard2_replica_t45] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_t45' (registry 'solr.core.collection1.shard2.replica_t45') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@367d25d9
   [junit4]   2> 499960 INFO  (qtp1064062937-4306) [n:127.0.0.1:56073_alaw c:collection1 s:shard2  x:collection1_shard2_replica_t45] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 499960 INFO  (qtp1064062937-4306) [n:127.0.0.1:56073_alaw c:collection1 s:shard2  x:collection1_shard2_replica_t45] o.a.s.c.SolrCore [[collection1_shard2_replica_t45] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_BE0EB204AEA5718D-001/shard-3-001/cores/collection1_shard2_replica_t45], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_BE0EB204AEA5718D-001/shard-3-001/cores/collection1_shard2_replica_t45/data/]
   [junit4]   2> 499964 INFO  (qtp1064062937-4306) [n:127.0.0.1:56073_alaw c:collection1 s:shard2  x:collection1_shard2_replica_t45] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=14, maxMergeAtOnceExplicit=40, maxMergedSegmentMB=26.4208984375, floorSegmentMB=0.791015625, forceMergeDeletesPctAllowed=17.283805331750646, segmentsPerTier=29.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 499967 WARN  (qtp1064062937-4306) [n:127.0.0.1:56073_alaw c:collection1 s:shard2  x:collection1_shard2_replica_t45] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 500017 INFO  (qtp1064062937-4306) [n:127.0.0.1:56073_alaw c:collection1 s:shard2  x:collection1_shard2_replica_t45] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 500017 INFO  (qtp1064062937-4306) [n:127.0.0.1:56073_alaw c:collection1 s:shard2  x:collection1_shard2_replica_t45] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 500024 INFO  (qtp1064062937-4306) [n:127.0.0.1:56073_alaw c:collection1 s:shard2  x:collection1_shard2_replica_t45] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 500024 INFO  (qtp1064062937-4306) [n:127.0.0.1:56073_alaw c:collection1 s:shard2  x:collection1_shard2_replica_t45] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 500025 INFO  (qtp1064062937-4306) [n:127.0.0.1:56073_alaw c:collection1 s:shard2  x:collection1_shard2_replica_t45] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=32, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 500026 INFO  (qtp1064062937-4306) [n:127.0.0.1:56073_alaw c:collection1 s:shard2  x:collection1_shard2_replica_t45] o.a.s.s.SolrIndexSearcher Opening [Searcher@37008716[collection1_shard2_replica_t45] main]
   [junit4]   2> 500027 INFO  (qtp1064062937-4306) [n:127.0.0.1:56073_alaw c:collection1 s:shard2  x:collection1_shard2_replica_t45] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 500028 INFO  (qtp1064062937-4306) [n:127.0.0.1:56073_alaw c:collection1 s:shard2  x:collection1_shard2_replica_t45] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 500028 INFO  (qtp1064062937-4306) [n:127.0.0.1:56073_alaw c:collection1 s:shard2  x:collection1_shard2_replica_t45] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 500028 INFO  (qtp1064062937-4306) [n:127.0.0.1:56073_alaw c:collection1 s:shard2  x:collection1_shard2_replica_t45] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1577296494578368512
   [junit4]   2> 500031 INFO  (searcherExecutor-1359-thread-1-processing-n:127.0.0.1:56073_alaw x:collection1_shard2_replica_t45 s:shard2 c:collection1) [n:127.0.0.1:56073_alaw c:collection1 s:shard2  x:collection1_shard2_replica_t45] o.a.s.c.SolrCore [collection1_shard2_replica_t45] Registered new searcher Searcher@37008716[collection1_shard2_replica_t45] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 500037 INFO  (qtp1064062937-4306) [n:127.0.0.1:56073_alaw c:collection1 s:shard2  x:collection1_shard2_replica_t45] o.a.s.c.ZkController Core needs to recover:collection1_shard2_replica_t45
   [junit4]   2> 500037 INFO  (updateExecutor-565-thread-1-processing-n:127.0.0.1:56073_alaw x:collection1_shard2_replica_t45 s:shard2 c:collection1) [n:127.0.0.1:56073_alaw c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_t45] o.a.s.u.DefaultSolrCoreState Running recovery
   [junit4]   2> 500037 INFO  (recoveryExecutor-566-thread-1-processing-n:127.0.0.1:56073_alaw x:collection1_shard2_replica_t45 s:shard2 c:collection1 r:core_node46) [n:127.0.0.1:56073_alaw c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_t45] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
   [junit4]   2> 500039 INFO  (recoveryExecutor-566-thread-1-processing-n:127.0.0.1:56073_alaw x:collection1_shard2_replica_t45 s:shard2 c:collection1 r:core_node46) [n:127.0.0.1:56073_alaw c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_t45] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
   [junit4]   2> 500039 INFO  (recoveryExecutor-566-thread-1-processing-n:127.0.0.1:56073_alaw x:collection1_shard2_replica_t45 s:shard2 c:collection1 r:core_node46) [n:127.0.0.1:56073_alaw c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_t45] o.a.s.c.ZkController collection1_shard2_replica_t45 stopping background replication from leader
   [junit4]   2> 500039 INFO  (qtp1064062937-4306) [n:127.0.0.1:56073_alaw c:collection1 s:shard2  x:collection1_shard2_replica_t45] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_t45&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=1224
   [junit4]   2> 500039 INFO  (recoveryExecutor-566-thread-1-processing-n:127.0.0.1:56073_alaw x:collection1_shard2_replica_t45 s:shard2 c:collection1 r:core_node46) [n:127.0.0.1:56073_alaw c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_t45] o.a.s.c.RecoveryStrategy Begin buffering updates. core=[collection1_shard2_replica_t45]
   [junit4]   2> 500039 INFO  (recoveryExecutor-566-thread-1-processing-n:127.0.0.1:56073_alaw x:collection1_shard2_replica_t45 s:shard2 c:collection1 r:core_node46) [n:127.0.0.1:56073_alaw c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_t45] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
   [junit4]   2> 500039 INFO  (recoveryExecutor-566-thread-1-processing-n:127.0.0.1:56073_alaw x:collection1_shard2_replica_t45 s:shard2 c:collection1 r:core_node46) [n:127.0.0.1:56073_alaw c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_t45] o.a.s.c.RecoveryStrategy Publishing state of core [collection1_shard2_replica_t45] as recovering, leader is [http://127.0.0.1:56053/alaw/collection1_shard2_replica_t41/] and I am [http://127.0.0.1:56073/alaw/collection1_shard2_replica_t45/]
   [junit4]   2> 500042 INFO  (qtp1459346964-4206) [n:127.0.0.1:56044_alaw    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:56073_alaw&action=ADDREPLICA&collection=collection1&shard=shard2&type=TLOG&wt=javabin&version=2} status=0 QTime=1240
   [junit4]   2> 500043 INFO  (recoveryExecutor-566-thread-1-processing-n:127.0.0.1:56073_alaw x:collection1_shard2_replica_t45 s:shard2 c:collection1 r:core_node46) [n:127.0.0.1:56073_alaw c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_t45] o.a.s.c.RecoveryStrategy Sending prep recovery command to [http://127.0.0.1:56053/alaw]; [WaitForState: action=PREPRECOVERY&core=collection1_shard2_replica_t41&nodeName=127.0.0.1:56073_alaw&coreNodeName=core_node46&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true]
   [junit4]   2> 500044 INFO  (qtp1503893309-4249) [n:127.0.0.1:56053_alaw    ] o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node46, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true, maxTime: 183 s
   [junit4]   2> 500045 INFO  (qtp1503893309-4249) [n:127.0.0.1:56053_alaw    ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard2, thisCore=collection1_shard2_replica_t41, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:56073_alaw, coreNodeName=core_node46, onlyIfActiveCheckResult=false, nodeProps: core_node46:{"core":"collection1_shard2_replica_t45","base_url":"http://127.0.0.1:56073/alaw","node_name":"127.0.0.1:56073_alaw","state":"down","type":"TLOG"}
   [junit4]   2> 500150 INFO  (zkCallback-568-thread-1-processing-n:127.0.0.1:56073_alaw) [n:127.0.0.1:56073_alaw    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 500151 INFO  (zkCallback-556-thread-1-processing-n:127.0.0.1:56053_alaw) [n:127.0.0.1:56053_alaw    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 500150 INFO  (zkCallback-562-thread-1-processing-n:127.0.0.1:56067_alaw) [n:127.0.0.1:56067_alaw    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConn

[...truncated too long message...]

953, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@643441b8
   [junit4]   2> 2957010 WARN  (zkCallback-4009-thread-2-processing-n:127.0.0.1:49954_solr) [n:127.0.0.1:49954_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 2957010 INFO  (jetty-closer-3980-thread-1) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@5874d06d{/solr,null,UNAVAILABLE}
   [junit4]   2> 2957010 INFO  (zkCallback-3991-thread-4-processing-n:127.0.0.1:49953_solr) [n:127.0.0.1:49953_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
   [junit4]   2> 2957012 INFO  (jetty-closer-3980-thread-2) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
   [junit4]   2> 2957013 INFO  (jetty-closer-3980-thread-2) [    ] o.a.s.c.Overseer Overseer (id=98581181711450118-127.0.0.1:49953_solr-n_0000000000) closing
   [junit4]   2> 2957013 INFO  (OverseerStateUpdate-98581181711450118-127.0.0.1:49953_solr-n_0000000000) [n:127.0.0.1:49953_solr    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:49953_solr
   [junit4]   2> 2957015 WARN  (zkCallback-3991-thread-4-processing-n:127.0.0.1:49953_solr) [n:127.0.0.1:49953_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 2957016 INFO  (jetty-closer-3980-thread-2) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@82d6289{/solr,null,UNAVAILABLE}
   [junit4]   2> 2957017 ERROR (SUITE-TestPullReplica-seed#[BE0EB204AEA5718D]-worker) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 2957017 INFO  (SUITE-TestPullReplica-seed#[BE0EB204AEA5718D]-worker) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:49951 49951
   [junit4]   2> 2957033 INFO  (Thread-3718) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:49951 49951
   [junit4]   2> 2958690 WARN  (Thread-3718) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	19	/solr/configs/conf
   [junit4]   2> 	5	/solr/aliases.json
   [junit4]   2> 	4	/solr/security.json
   [junit4]   2> 	3	/solr/clusterprops.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	26	/solr/collections/pull_replica_test_create_delete/state.json
   [junit4]   2> 	24	/solr/collections/pull_replica_test_kill_pull_replica/state.json
   [junit4]   2> 	21	/solr/collections/pull_replica_test_add_remove_pull_replica/state.json
   [junit4]   2> 	20	/solr/collections/pull_replica_test_kill_leader/state.json
   [junit4]   2> 	20	/solr/collections/pull_replica_test_remove_all_writer_replicas/state.json
   [junit4]   2> 	13	/solr/collections/pull_replica_test_add_docs/state.json
   [junit4]   2> 	13	/solr/collections/pull_replica_test_real_time_get/state.json
   [junit4]   2> 	5	/solr/clusterstate.json
   [junit4]   2> 	5	/solr/clusterprops.json
   [junit4]   2> 	3	/solr/overseer_elect/election/98581181711450118-127.0.0.1:49953_solr-n_0000000000
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	5	/solr/collections
   [junit4]   2> 	3	/solr/live_nodes
   [junit4]   2> 
   [junit4]   2> NOTE: leaving temporary files on disk at: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.TestPullReplica_BE0EB204AEA5718D-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {foo=PostingsFormat(name=Memory), foo_s=PostingsFormat(name=Memory), id=Lucene50(blocksize=128)}, docValues:{}, maxPointsInLeafNode=841, maxMBSortInHeap=6.684027924152215, sim=RandomSimilarity(queryNorm=true): {}, locale=it-IT, timezone=Pacific/Enderbury
   [junit4]   2> NOTE: Mac OS X 10.11.6 x86_64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=3,threads=1,free=325786216,total=518979584
   [junit4]   2> NOTE: All tests run in this JVM: [TestAuthorizationFramework, TestSolrDeletionPolicy2, BlobRepositoryCloudTest, BlockJoinFacetSimpleTest, TestFieldCacheSortRandom, DistributedVersionInfoTest, TestNRTOpen, UnloadDistributedZkTest, TestCopyFieldCollectionResource, TestStressRecovery, AsyncCallRequestStatusResponseTest, TestSchemaManager, TestCollapseQParserPlugin, LeaderElectionTest, TestCorePropertiesReload, TestSolrCLIRunExample, RequiredFieldsTest, CollectionsAPISolrJTest, DistributedFacetExistsSmallTest, TestDynamicLoading, RuleEngineTest, AnalyticsMergeStrategyTest, DistanceUnitsTest, CdcrVersionReplicationTest, TestMinMaxOnMultiValuedField, ZkControllerTest, BadCopyFieldTest, CdcrReplicationHandlerTest, TestGroupingSearch, DeleteNodeTest, TestRebalanceLeaders, TestComplexPhraseQParserPlugin, TestReloadDeadlock, TestNumericRangeQuery64, BigEndianAscendingWordDeserializerTest, DirectUpdateHandlerOptimizeTest, CollectionReloadTest, BlockJoinFacetDistribTest, ClusterStateUpdateTest, OverseerModifyCollectionTest, SolrGraphiteReporterTest, CoreSorterTest, ConfigSetsAPITest, TestQuerySenderListener, CircularListTest, ForceLeaderTest, TestObjectReleaseTracker, TestNumericTerms32, DistributedFacetPivotSmallTest, TestXmlQParser, TestLegacyField, UpdateLogTest, TestAuthenticationFramework, SimpleFacetsTest, ClusterStateTest, ShardRoutingTest, TestLeaderElectionWithEmptyReplica, TestJsonFacetRefinement, HighlighterTest, ResponseLogComponentTest, TestReversedWildcardFilterFactory, NumericFieldsTest, DateRangeFieldTest, CollectionTooManyReplicasTest, TestHighlightDedupGrouping, DefaultValueUpdateProcessorTest, CursorPagingTest, TestGraphMLResponseWriter, OverriddenZkACLAndCredentialsProvidersTest, ConjunctionSolrSpellCheckerTest, TestFieldTypeResource, TestClusterStateMutator, SyncSliceTest, CursorMarkTest, TestAtomicUpdateErrorCases, BinaryUpdateRequestHandlerTest, CdcrBootstrapTest, TestPerFieldSimilarityWithDefaultOverride, TemplateUpdateProcessorTest, DistributedSuggestComponentTest, SuggestComponentContextFilterQueryTest, OverseerStatusTest, SolrMetricManagerTest, RegexBoostProcessorTest, UUIDFieldTest, TestFieldResource, HdfsDirectoryTest, TestDefaultStatsCache, TestFileDictionaryLookup, MoveReplicaTest, TestLazyCores, DirectSolrConnectionTest, HighlighterMaxOffsetTest, ZkFailoverTest, TestPHPSerializedResponseWriter, TestQuerySenderNoQuery, TestTolerantSearch, TestSort, ZkStateWriterTest, TestCoreContainer, TestShortCircuitedRequests, HttpPartitionTest, DirectUpdateHandlerTest, TestJettySolrRunner, TestDistributedSearch, MoveReplicaHDFSTest, ExitableDirectoryReaderTest, TestApiFramework, TestHttpShardHandlerFactory, SuggesterTest, TestSchemaResource, RemoteQueryErrorTest, TestHashPartitioner, TestMultiWordSynonyms, LeaderInitiatedRecoveryOnCommitTest, UtilsToolTest, DeleteInactiveReplicaTest, HdfsBasicDistributedZkTest, IgnoreCommitOptimizeUpdateProcessorFactoryTest, PrimitiveFieldTypeTest, SimpleMLTQParserTest, V2StandaloneTest, TestFieldSortValues, TestOverriddenPrefixQueryForCustomFieldType, XsltUpdateRequestHandlerTest, CloneFieldUpdateProcessorFactoryTest, TestIndexingPerformance, TestRequestForwarding, TestReqParamsAPI, DistanceFunctionTest, TestCloudManagedSchema, MergeStrategyTest, SegmentsInfoRequestHandlerTest, TestCursorMarkWithoutUniqueKey, TestCloudPivotFacet, ZkCLITest, TestOmitPositions, TestSystemCollAutoCreate, DistributedDebugComponentTest, TestSolrConfigHandlerCloud, TestEmbeddedSolrServerConstructors, MigrateRouteKeyTest, URLClassifyProcessorTest, TestGraphTermsQParserPlugin, RequestHandlersTest, HdfsChaosMonkeySafeLeaderTest, TestUniqueKeyFieldResource, TestRuleBasedAuthorizationPlugin, TestFieldCacheReopen, WordBreakSolrSpellCheckerTest, IndexBasedSpellCheckerTest, PropertiesRequestHandlerTest, TestMultiValuedNumericRangeQuery, TestXIncludeConfig, TestFuzzyAnalyzedSuggestions, TriLevelCompositeIdRoutingTest, TestIntervalFaceting, BasicDistributedZkTest, ScriptEngineTest, TestCodecSupport, JSONWriterTest, TestSolrCloudSnapshots, SortByFunctionTest, TestSchemaVersionResource, TestDistribDocBasedVersion, DistributedQueryComponentCustomSortTest, TestLRUStatsCache, RAMDirectoryFactoryTest, DisMaxRequestHandlerTest, TestSolrCloudWithSecureImpersonation, TestDocTermOrds, TestStressInPlaceUpdates, DataDrivenBlockJoinTest, TestPseudoReturnFields, TestFoldingMultitermQuery, TestJmxIntegration, TestManagedResource, SolrCloudReportersTest, TestIBSimilarityFactory, PeerSyncReplicationTest, DateFieldTest, ChaosMonkeySafeLeaderWithPullReplicasTest, ConnectionManagerTest, CreateCollectionCleanupTest, DeleteShardTest, DistribCursorPagingTest, DistribJoinFromCollectionTest, LeaderElectionContextKeyTest, MoveReplicaHDFSUlogDirTest, NodeMutatorTest, OverseerRolesTest, RecoveryAfterSoftCommitTest, SSLMigrationTest, SaslZkACLProviderTest, SolrCloudExampleTest, TestCloudSearcherWarming, TestConfigSetsAPI, TestLocalFSCloudBackupRestore, TestPullReplica]
   [junit4] Completed [539/732 (2!)] on J0 in 157.86s, 10 tests, 1 failure, 1 skipped <<< FAILURES!

[...truncated 46755 lines...]
BUILD FAILED
/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/build.xml:810: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/build.xml:690: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/build.xml:678: Source checkout is modified!!! Offending files:
* solr/licenses/httpmime-4.5.3.jar.sha1

Total time: 94 minutes 40 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any

[JENKINS] Lucene-Solr-master-MacOSX (64bit/jdk1.8.0) - Build # 4159 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-MacOSX/4159/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseG1GC

2 tests failed.
FAILED:  org.apache.solr.cloud.LeaderElectionContextKeyTest.test

Error Message:


Stack Trace:
java.lang.AssertionError
	at __randomizedtesting.SeedInfo.seed([352ADDAB357047D5:BD7EE2719B8C2A2D]:0)
	at org.junit.Assert.fail(Assert.java:92)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertTrue(Assert.java:54)
	at org.apache.solr.cloud.LeaderElectionContextKeyTest.test(LeaderElectionContextKeyTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.ZkFailoverTest.testRestartZkWhenClusterDown

Error Message:
Collection not found: coll1

Stack Trace:
org.apache.solr.common.SolrException: Collection not found: coll1
	at __randomizedtesting.SeedInfo.seed([352ADDAB357047D5:5BBD5EE74C97E156]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.getCollectionNames(CloudSolrClient.java:1139)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:822)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:173)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:138)
	at org.apache.solr.cloud.ZkFailoverTest.testRestartZkWhenClusterDown(ZkFailoverTest.java:50)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 12804 lines...]
   [junit4] Suite: org.apache.solr.cloud.LeaderElectionContextKeyTest
   [junit4]   2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderElectionContextKeyTest_352ADDAB357047D5-001/init-core-data-001
   [junit4]   2> 1985134 WARN  (SUITE-LeaderElectionContextKeyTest-seed#[352ADDAB357047D5]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=105 numCloses=105
   [junit4]   2> 1985135 INFO  (SUITE-LeaderElectionContextKeyTest-seed#[352ADDAB357047D5]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 1985139 INFO  (SUITE-LeaderElectionContextKeyTest-seed#[352ADDAB357047D5]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN) w/ MAC_OS_X supressed clientAuth
   [junit4]   2> 1985139 INFO  (SUITE-LeaderElectionContextKeyTest-seed#[352ADDAB357047D5]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 1985140 INFO  (SUITE-LeaderElectionContextKeyTest-seed#[352ADDAB357047D5]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 1 servers in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderElectionContextKeyTest_352ADDAB357047D5-001/tempDir-001
   [junit4]   2> 1985141 INFO  (SUITE-LeaderElectionContextKeyTest-seed#[352ADDAB357047D5]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 1985141 INFO  (Thread-3318) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1985141 INFO  (Thread-3318) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 1985144 ERROR (Thread-3318) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 1985248 INFO  (SUITE-LeaderElectionContextKeyTest-seed#[352ADDAB357047D5]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:61521
   [junit4]   2> 1985338 INFO  (jetty-launcher-3575-thread-1) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 1985340 INFO  (jetty-launcher-3575-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6f8c08aa{/solr,null,AVAILABLE}
   [junit4]   2> 1985360 INFO  (jetty-launcher-3575-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@391438f3{HTTP/1.1,[http/1.1]}{127.0.0.1:61523}
   [junit4]   2> 1985363 INFO  (jetty-launcher-3575-thread-1) [    ] o.e.j.s.Server Started @1989595ms
   [junit4]   2> 1985363 INFO  (jetty-launcher-3575-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=61523}
   [junit4]   2> 1985364 ERROR (jetty-launcher-3575-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1985364 INFO  (jetty-launcher-3575-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 1985364 INFO  (jetty-launcher-3575-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1985364 INFO  (jetty-launcher-3575-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 1985364 INFO  (jetty-launcher-3575-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-08-31T14:17:46.733Z
   [junit4]   2> 1985388 INFO  (jetty-launcher-3575-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1985405 INFO  (jetty-launcher-3575-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:61521/solr
   [junit4]   2> 1985920 INFO  (jetty-launcher-3575-thread-1) [n:127.0.0.1:61523_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1985921 INFO  (jetty-launcher-3575-thread-1) [n:127.0.0.1:61523_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:61523_solr
   [junit4]   2> 1985923 INFO  (jetty-launcher-3575-thread-1) [n:127.0.0.1:61523_solr    ] o.a.s.c.Overseer Overseer (id=98578534662995971-127.0.0.1:61523_solr-n_0000000000) starting
   [junit4]   2> 1985934 INFO  (jetty-launcher-3575-thread-1) [n:127.0.0.1:61523_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:61523_solr
   [junit4]   2> 1985937 INFO  (zkCallback-3582-thread-1-processing-n:127.0.0.1:61523_solr) [n:127.0.0.1:61523_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1986206 INFO  (jetty-launcher-3575-thread-1) [n:127.0.0.1:61523_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_61523.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7da5c4cc
   [junit4]   2> 1986213 INFO  (jetty-launcher-3575-thread-1) [n:127.0.0.1:61523_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_61523.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7da5c4cc
   [junit4]   2> 1986213 INFO  (jetty-launcher-3575-thread-1) [n:127.0.0.1:61523_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_61523.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7da5c4cc
   [junit4]   2> 1986215 INFO  (jetty-launcher-3575-thread-1) [n:127.0.0.1:61523_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderElectionContextKeyTest_352ADDAB357047D5-001/tempDir-001/node1/.
   [junit4]   2> 1986249 INFO  (SUITE-LeaderElectionContextKeyTest-seed#[352ADDAB357047D5]-worker) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1986251 INFO  (SUITE-LeaderElectionContextKeyTest-seed#[352ADDAB357047D5]-worker) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:61521/solr ready
   [junit4]   2> 1986270 INFO  (qtp896082360-20404) [n:127.0.0.1:61523_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=config&maxShardsPerNode=1000&name=testCollection1&nrtReplicas=1&action=CREATE&numShards=2&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1986273 INFO  (OverseerThreadFactory-8552-thread-1-processing-n:127.0.0.1:61523_solr) [n:127.0.0.1:61523_solr    ] o.a.s.c.CreateCollectionCmd Create collection testCollection1
   [junit4]   2> 1986398 INFO  (OverseerStateUpdate-98578534662995971-127.0.0.1:61523_solr-n_0000000000) [n:127.0.0.1:61523_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"testCollection1",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"testCollection1_shard1_replica_n1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:61523/solr",
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 1986409 INFO  (OverseerStateUpdate-98578534662995971-127.0.0.1:61523_solr-n_0000000000) [n:127.0.0.1:61523_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"testCollection1",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"testCollection1_shard2_replica_n3",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:61523/solr",
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 1986620 INFO  (qtp896082360-20405) [n:127.0.0.1:61523_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node4&collection.configName=config&newCollection=true&name=testCollection1_shard2_replica_n3&action=CREATE&numShards=2&collection=testCollection1&shard=shard2&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 1986621 INFO  (qtp896082360-20408) [n:127.0.0.1:61523_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node2&collection.configName=config&newCollection=true&name=testCollection1_shard1_replica_n1&action=CREATE&numShards=2&collection=testCollection1&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 1986622 INFO  (qtp896082360-20408) [n:127.0.0.1:61523_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 1986754 INFO  (zkCallback-3582-thread-1-processing-n:127.0.0.1:61523_solr) [n:127.0.0.1:61523_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCollection1/state.json] for collection [testCollection1] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1986755 INFO  (zkCallback-3582-thread-2-processing-n:127.0.0.1:61523_solr) [n:127.0.0.1:61523_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCollection1/state.json] for collection [testCollection1] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1986756 INFO  (zkCallback-3582-thread-3-processing-n:127.0.0.1:61523_solr) [n:127.0.0.1:61523_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCollection1/state.json] for collection [testCollection1] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1987660 INFO  (qtp896082360-20405) [n:127.0.0.1:61523_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 1987661 INFO  (qtp896082360-20408) [n:127.0.0.1:61523_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 1987672 INFO  (qtp896082360-20405) [n:127.0.0.1:61523_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.s.IndexSchema [testCollection1_shard2_replica_n3] Schema name=minimal
   [junit4]   2> 1987707 INFO  (qtp896082360-20408) [n:127.0.0.1:61523_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.s.IndexSchema [testCollection1_shard1_replica_n1] Schema name=minimal
   [junit4]   2> 1987708 INFO  (qtp896082360-20405) [n:127.0.0.1:61523_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 1987708 INFO  (qtp896082360-20405) [n:127.0.0.1:61523_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.c.CoreContainer Creating SolrCore 'testCollection1_shard2_replica_n3' using configuration from collection testCollection1, trusted=true
   [junit4]   2> 1987710 INFO  (qtp896082360-20405) [n:127.0.0.1:61523_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_61523.solr.core.testCollection1.shard2.replica_n3' (registry 'solr.core.testCollection1.shard2.replica_n3') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7da5c4cc
   [junit4]   2> 1987711 INFO  (qtp896082360-20405) [n:127.0.0.1:61523_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1987711 INFO  (qtp896082360-20405) [n:127.0.0.1:61523_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.c.SolrCore [[testCollection1_shard2_replica_n3] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderElectionContextKeyTest_352ADDAB357047D5-001/tempDir-001/node1/testCollection1_shard2_replica_n3], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderElectionContextKeyTest_352ADDAB357047D5-001/tempDir-001/node1/./testCollection1_shard2_replica_n3/data/]
   [junit4]   2> 1987711 INFO  (qtp896082360-20408) [n:127.0.0.1:61523_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 1987711 INFO  (qtp896082360-20408) [n:127.0.0.1:61523_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'testCollection1_shard1_replica_n1' using configuration from collection testCollection1, trusted=true
   [junit4]   2> 1987712 INFO  (qtp896082360-20408) [n:127.0.0.1:61523_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_61523.solr.core.testCollection1.shard1.replica_n1' (registry 'solr.core.testCollection1.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7da5c4cc
   [junit4]   2> 1987712 INFO  (qtp896082360-20408) [n:127.0.0.1:61523_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1987712 INFO  (qtp896082360-20408) [n:127.0.0.1:61523_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.c.SolrCore [[testCollection1_shard1_replica_n1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderElectionContextKeyTest_352ADDAB357047D5-001/tempDir-001/node1/testCollection1_shard1_replica_n1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderElectionContextKeyTest_352ADDAB357047D5-001/tempDir-001/node1/./testCollection1_shard1_replica_n1/data/]
   [junit4]   2> 1987824 INFO  (qtp896082360-20405) [n:127.0.0.1:61523_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1987824 INFO  (qtp896082360-20405) [n:127.0.0.1:61523_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1987826 INFO  (qtp896082360-20408) [n:127.0.0.1:61523_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1987826 INFO  (qtp896082360-20408) [n:127.0.0.1:61523_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1987828 INFO  (qtp896082360-20405) [n:127.0.0.1:61523_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1987869 INFO  (qtp896082360-20405) [n:127.0.0.1:61523_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1987873 INFO  (qtp896082360-20405) [n:127.0.0.1:61523_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.s.SolrIndexSearcher Opening [Searcher@26a905d3[testCollection1_shard2_replica_n3] main]
   [junit4]   2> 1987875 INFO  (qtp896082360-20408) [n:127.0.0.1:61523_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1987875 INFO  (qtp896082360-20408) [n:127.0.0.1:61523_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1987877 INFO  (qtp896082360-20405) [n:127.0.0.1:61523_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/config
   [junit4]   2> 1987879 INFO  (qtp896082360-20405) [n:127.0.0.1:61523_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/config
   [junit4]   2> 1987880 INFO  (qtp896082360-20408) [n:127.0.0.1:61523_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@2921a2b4[testCollection1_shard1_replica_n1] main]
   [junit4]   2> 1987882 INFO  (qtp896082360-20405) [n:127.0.0.1:61523_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1987882 INFO  (qtp896082360-20405) [n:127.0.0.1:61523_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1577256557478936576
   [junit4]   2> 1987883 INFO  (searcherExecutor-8555-thread-1-processing-n:127.0.0.1:61523_solr x:testCollection1_shard2_replica_n3 s:shard2 c:testCollection1 r:core_node4) [n:127.0.0.1:61523_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.c.SolrCore [testCollection1_shard2_replica_n3] Registered new searcher Searcher@26a905d3[testCollection1_shard2_replica_n3] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1987884 INFO  (qtp896082360-20408) [n:127.0.0.1:61523_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/config
   [junit4]   2> 1987884 INFO  (qtp896082360-20408) [n:127.0.0.1:61523_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/config
   [junit4]   2> 1987888 INFO  (qtp896082360-20408) [n:127.0.0.1:61523_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1987888 INFO  (qtp896082360-20408) [n:127.0.0.1:61523_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1577256557485228032
   [junit4]   2> 1987889 INFO  (searcherExecutor-8556-thread-1-processing-n:127.0.0.1:61523_solr x:testCollection1_shard1_replica_n1 s:shard1 c:testCollection1 r:core_node2) [n:127.0.0.1:61523_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.c.SolrCore [testCollection1_shard1_replica_n1] Registered new searcher Searcher@2921a2b4[testCollection1_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1987892 INFO  (qtp896082360-20405) [n:127.0.0.1:61523_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1987893 INFO  (qtp896082360-20405) [n:127.0.0.1:61523_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1987894 INFO  (qtp896082360-20405) [n:127.0.0.1:61523_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:61523/solr/testCollection1_shard2_replica_n3/
   [junit4]   2> 1987895 INFO  (qtp896082360-20405) [n:127.0.0.1:61523_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 1987895 INFO  (qtp896082360-20405) [n:127.0.0.1:61523_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.c.SyncStrategy http://127.0.0.1:61523/solr/testCollection1_shard2_replica_n3/ has no replicas
   [junit4]   2> 1987895 INFO  (qtp896082360-20405) [n:127.0.0.1:61523_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 1987901 INFO  (qtp896082360-20408) [n:127.0.0.1:61523_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1987901 INFO  (qtp896082360-20408) [n:127.0.0.1:61523_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1987901 INFO  (qtp896082360-20408) [n:127.0.0.1:61523_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:61523/solr/testCollection1_shard1_replica_n1/
   [junit4]   2> 1987901 INFO  (qtp896082360-20408) [n:127.0.0.1:61523_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 1987901 INFO  (qtp896082360-20408) [n:127.0.0.1:61523_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:61523/solr/testCollection1_shard1_replica_n1/ has no replicas
   [junit4]   2> 1987901 INFO  (qtp896082360-20408) [n:127.0.0.1:61523_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 1987909 INFO  (qtp896082360-20408) [n:127.0.0.1:61523_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:61523/solr/testCollection1_shard1_replica_n1/ shard1
   [junit4]   2> 1987914 INFO  (qtp896082360-20405) [n:127.0.0.1:61523_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:61523/solr/testCollection1_shard2_replica_n3/ shard2
   [junit4]   2> 1988023 INFO  (zkCallback-3582-thread-3-processing-n:127.0.0.1:61523_solr) [n:127.0.0.1:61523_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCollection1/state.json] for collection [testCollection1] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1988023 INFO  (zkCallback-3582-thread-1-processing-n:127.0.0.1:61523_solr) [n:127.0.0.1:61523_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCollection1/state.json] for collection [testCollection1] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1988023 INFO  (zkCallback-3582-thread-2-processing-n:127.0.0.1:61523_solr) [n:127.0.0.1:61523_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCollection1/state.json] for collection [testCollection1] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1988024 INFO  (qtp896082360-20405) [n:127.0.0.1:61523_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 1988025 INFO  (qtp896082360-20405) [n:127.0.0.1:61523_solr c:testCollection1 s:shard2 r:core_node4 x:testCollection1_shard2_replica_n3] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node4&collection.configName=config&newCollection=true&name=testCollection1_shard2_replica_n3&action=CREATE&numShards=2&collection=testCollection1&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1405
   [junit4]   2> 1988081 INFO  (qtp896082360-20408) [n:127.0.0.1:61523_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 1988083 INFO  (qtp896082360-20408) [n:127.0.0.1:61523_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node2&collection.configName=config&newCollection=true&name=testCollection1_shard1_replica_n1&action=CREATE&numShards=2&collection=testCollection1&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1462
   [junit4]   2> 1988090 INFO  (qtp896082360-20404) [n:127.0.0.1:61523_solr    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 1988192 INFO  (zkCallback-3582-thread-2-processing-n:127.0.0.1:61523_solr) [n:127.0.0.1:61523_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCollection1/state.json] for collection [testCollection1] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1988192 INFO  (zkCallback-3582-thread-1-processing-n:127.0.0.1:61523_solr) [n:127.0.0.1:61523_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCollection1/state.json] for collection [testCollection1] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1988197 INFO  (zkCallback-3582-thread-3-processing-n:127.0.0.1:61523_solr) [n:127.0.0.1:61523_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCollection1/state.json] for collection [testCollection1] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1988287 INFO  (OverseerCollectionConfigSetProcessor-98578534662995971-127.0.0.1:61523_solr-n_0000000000) [n:127.0.0.1:61523_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1989093 INFO  (qtp896082360-20404) [n:127.0.0.1:61523_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=config&maxShardsPerNode=1000&name=testCollection1&nrtReplicas=1&action=CREATE&numShards=2&wt=javabin&version=2} status=0 QTime=2822
   [junit4]   2> 1989094 INFO  (qtp896082360-20409) [n:127.0.0.1:61523_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=config&maxShardsPerNode=1000&name=testCollection2&nrtReplicas=1&action=CREATE&numShards=2&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1989098 INFO  (OverseerThreadFactory-8552-thread-2-processing-n:127.0.0.1:61523_solr) [n:127.0.0.1:61523_solr    ] o.a.s.c.CreateCollectionCmd Create collection testCollection2
   [junit4]   2> 1989317 INFO  (OverseerStateUpdate-98578534662995971-127.0.0.1:61523_solr-n_0000000000) [n:127.0.0.1:61523_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"testCollection2",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"testCollection2_shard1_replica_n1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:61523/solr",
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 1989328 INFO  (OverseerStateUpdate-98578534662995971-127.0.0.1:61523_solr-n_0000000000) [n:127.0.0.1:61523_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"testCollection2",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"testCollection2_shard2_replica_n2",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:61523/solr",
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 1989533 INFO  (qtp896082360-20402) [n:127.0.0.1:61523_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node3&collection.configName=config&newCollection=true&name=testCollection2_shard1_replica_n1&action=CREATE&numShards=2&collection=testCollection2&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 1989533 INFO  (qtp896082360-20406) [n:127.0.0.1:61523_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node4&collection.configName=config&newCollection=true&name=testCollection2_shard2_replica_n2&action=CREATE&numShards=2&collection=testCollection2&shard=shard2&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 1989646 INFO  (zkCallback-3582-thread-1-processing-n:127.0.0.1:61523_solr) [n:127.0.0.1:61523_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCollection2/state.json] for collection [testCollection2] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1989646 INFO  (zkCallback-3582-thread-3-processing-n:127.0.0.1:61523_solr) [n:127.0.0.1:61523_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCollection2/state.json] for collection [testCollection2] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1990578 INFO  (qtp896082360-20402) [n:127.0.0.1:61523_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 1990584 INFO  (qtp896082360-20406) [n:127.0.0.1:61523_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 1990592 INFO  (qtp896082360-20406) [n:127.0.0.1:61523_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.s.IndexSchema [testCollection2_shard2_replica_n2] Schema name=minimal
   [junit4]   2> 1990596 INFO  (qtp896082360-20406) [n:127.0.0.1:61523_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 1990596 INFO  (qtp896082360-20406) [n:127.0.0.1:61523_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.c.CoreContainer Creating SolrCore 'testCollection2_shard2_replica_n2' using configuration from collection testCollection2, trusted=true
   [junit4]   2> 1990596 INFO  (qtp896082360-20406) [n:127.0.0.1:61523_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_61523.solr.core.testCollection2.shard2.replica_n2' (registry 'solr.core.testCollection2.shard2.replica_n2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7da5c4cc
   [junit4]   2> 1990596 INFO  (qtp896082360-20406) [n:127.0.0.1:61523_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1990596 INFO  (qtp896082360-20406) [n:127.0.0.1:61523_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.c.SolrCore [[testCollection2_shard2_replica_n2] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderElectionContextKeyTest_352ADDAB357047D5-001/tempDir-001/node1/testCollection2_shard2_replica_n2], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderElectionContextKeyTest_352ADDAB357047D5-001/tempDir-001/node1/./testCollection2_shard2_replica_n2/data/]
   [junit4]   2> 1990599 INFO  (qtp896082360-20402) [n:127.0.0.1:61523_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.s.IndexSchema [testCollection2_shard1_replica_n1] Schema name=minimal
   [junit4]   2> 1990606 INFO  (qtp896082360-20402) [n:127.0.0.1:61523_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 1990606 INFO  (qtp896082360-20402) [n:127.0.0.1:61523_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'testCollection2_shard1_replica_n1' using configuration from collection testCollection2, trusted=true
   [junit4]   2> 1990607 INFO  (qtp896082360-20402) [n:127.0.0.1:61523_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_61523.solr.core.testCollection2.shard1.replica_n1' (registry 'solr.core.testCollection2.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7da5c4cc
   [junit4]   2> 1990607 INFO  (qtp896082360-20402) [n:127.0.0.1:61523_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1990607 INFO  (qtp896082360-20402) [n:127.0.0.1:61523_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.c.SolrCore [[testCollection2_shard1_replica_n1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderElectionContextKeyTest_352ADDAB357047D5-001/tempDir-001/node1/testCollection2_shard1_replica_n1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderElectionContextKeyTest_352ADDAB357047D5-001/tempDir-001/node1/./testCollection2_shard1_replica_n1/data/]
   [junit4]   2> 1990687 INFO  (qtp896082360-20406) [n:127.0.0.1:61523_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1990687 INFO  (qtp896082360-20406) [n:127.0.0.1:61523_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1990689 INFO  (qtp896082360-20406) [n:127.0.0.1:61523_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1990689 INFO  (qtp896082360-20406) [n:127.0.0.1:61523_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1990689 INFO  (qtp896082360-20402) [n:127.0.0.1:61523_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1990689 INFO  (qtp896082360-20402) [n:127.0.0.1:61523_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1990689 INFO  (qtp896082360-20406) [n:127.0.0.1:61523_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.s.SolrIndexSearcher Opening [Searcher@24be56e[testCollection2_shard2_replica_n2] main]
   [junit4]   2> 1990694 INFO  (qtp896082360-20406) [n:127.0.0.1:61523_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/config
   [junit4]   2> 1990695 INFO  (qtp896082360-20406) [n:127.0.0.1:61523_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/config
   [junit4]   2> 1990695 INFO  (qtp896082360-20402) [n:127.0.0.1:61523_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1990695 INFO  (qtp896082360-20402) [n:127.0.0.1:61523_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1990696 INFO  (qtp896082360-20406) [n:127.0.0.1:61523_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1990696 INFO  (qtp896082360-20406) [n:127.0.0.1:61523_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1577256560429629440
   [junit4]   2> 1990696 INFO  (searcherExecutor-8565-thread-1-processing-n:127.0.0.1:61523_solr x:testCollection2_shard2_replica_n2 s:shard2 c:testCollection2 r:core_node4) [n:127.0.0.1:61523_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.c.SolrCore [testCollection2_shard2_replica_n2] Registered new searcher Searcher@24be56e[testCollection2_shard2_replica_n2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1990697 INFO  (qtp896082360-20402) [n:127.0.0.1:61523_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@34aaff21[testCollection2_shard1_replica_n1] main]
   [junit4]   2> 1990699 INFO  (qtp896082360-20402) [n:127.0.0.1:61523_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/config
   [junit4]   2> 1990700 INFO  (qtp896082360-20402) [n:127.0.0.1:61523_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/config
   [junit4]   2> 1990701 INFO  (qtp896082360-20402) [n:127.0.0.1:61523_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1990702 INFO  (searcherExecutor-8566-thread-1-processing-n:127.0.0.1:61523_solr x:testCollection2_shard1_replica_n1 s:shard1 c:testCollection2 r:core_node3) [n:127.0.0.1:61523_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.c.SolrCore [testCollection2_shard1_replica_n1] Registered new searcher Searcher@34aaff21[testCollection2_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1990702 INFO  (qtp896082360-20402) [n:127.0.0.1:61523_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1577256560435920896
   [junit4]   2> 1990706 INFO  (qtp896082360-20406) [n:127.0.0.1:61523_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1990706 INFO  (qtp896082360-20406) [n:127.0.0.1:61523_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1990706 INFO  (qtp896082360-20406) [n:127.0.0.1:61523_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:61523/solr/testCollection2_shard2_replica_n2/
   [junit4]   2> 1990707 INFO  (qtp896082360-20406) [n:127.0.0.1:61523_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 1990707 INFO  (qtp896082360-20406) [n:127.0.0.1:61523_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.c.SyncStrategy http://127.0.0.1:61523/solr/testCollection2_shard2_replica_n2/ has no replicas
   [junit4]   2> 1990707 INFO  (qtp896082360-20406) [n:127.0.0.1:61523_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 1990713 INFO  (qtp896082360-20402) [n:127.0.0.1:61523_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1990713 INFO  (qtp896082360-20402) [n:127.0.0.1:61523_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1990714 INFO  (qtp896082360-20402) [n:127.0.0.1:61523_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:61523/solr/testCollection2_shard1_replica_n1/
   [junit4]   2> 1990714 INFO  (qtp896082360-20402) [n:127.0.0.1:61523_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 1990714 INFO  (qtp896082360-20402) [n:127.0.0.1:61523_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:61523/solr/testCollection2_shard1_replica_n1/ has no replicas
   [junit4]   2> 1990714 INFO  (qtp896082360-20402) [n:127.0.0.1:61523_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 1990715 INFO  (qtp896082360-20406) [n:127.0.0.1:61523_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:61523/solr/testCollection2_shard2_replica_n2/ shard2
   [junit4]   2> 1990720 INFO  (qtp896082360-20402) [n:127.0.0.1:61523_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:61523/solr/testCollection2_shard1_replica_n1/ shard1
   [junit4]   2> 1990829 INFO  (zkCallback-3582-thread-3-processing-n:127.0.0.1:61523_solr) [n:127.0.0.1:61523_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCollection2/state.json] for collection [testCollection2] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1990829 INFO  (zkCallback-3582-thread-1-processing-n:127.0.0.1:61523_solr) [n:127.0.0.1:61523_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCollection2/state.json] for collection [testCollection2] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1990929 INFO  (qtp896082360-20406) [n:127.0.0.1:61523_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 1990930 INFO  (qtp896082360-20402) [n:127.0.0.1:61523_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 1990936 INFO  (qtp896082360-20406) [n:127.0.0.1:61523_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node4&collection.configName=config&newCollection=true&name=testCollection2_shard2_replica_n2&action=CREATE&numShards=2&collection=testCollection2&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1403
   [junit4]   2> 1990940 INFO  (qtp896082360-20402) [n:127.0.0.1:61523_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=config&newCollection=true&name=testCollection2_shard1_replica_n1&action=CREATE&numShards=2&collection=testCollection2&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1407
   [junit4]   2> 1990947 INFO  (qtp896082360-20409) [n:127.0.0.1:61523_solr    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 1991048 INFO  (zkCallback-3582-thread-1-processing-n:127.0.0.1:61523_solr) [n:127.0.0.1:61523_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCollection2/state.json] for collection [testCollection2] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1991048 INFO  (zkCallback-3582-thread-3-processing-n:127.0.0.1:61523_solr) [n:127.0.0.1:61523_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testCollection2/state.json] for collection [testCollection2] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1991103 INFO  (OverseerCollectionConfigSetProcessor-98578534662995971-127.0.0.1:61523_solr-n_0000000000) [n:127.0.0.1:61523_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1991957 INFO  (qtp896082360-20409) [n:127.0.0.1:61523_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=config&maxShardsPerNode=1000&name=testCollection2&nrtReplicas=1&action=CREATE&numShards=2&wt=javabin&version=2} status=0 QTime=2863
   [junit4]   2> 1991958 INFO  (SUITE-LeaderElectionContextKeyTest-seed#[352ADDAB357047D5]-worker) [    ] o.a.s.c.AbstractDistribZkTestBase Wait for recoveries to finish - collection: testCollection1 failOnTimeout:true timeout (sec):30
   [junit4]   2> 1991958 INFO  (SUITE-LeaderElectionContextKeyTest-seed#[352ADDAB357047D5]-worker) [    ] o.a.s.c.AbstractDistribZkTestBase Recoveries finished - collection: testCollection1
   [junit4]   2> 1991959 INFO  (SUITE-LeaderElectionContextKeyTest-seed#[352ADDAB357047D5]-worker) [    ] o.a.s.c.AbstractDistribZkTestBase Wait for recoveries to finish - collection: testCollection2 failOnTimeout:true timeout (sec):30
   [junit4]   2> 1991959 INFO  (SUITE-LeaderElectionContextKeyTest-seed#[352ADDAB357047D5]-worker) [    ] o.a.s.c.AbstractDistribZkTestBase Recoveries finished - collection: testCollection2
   [junit4]   2> 1992103 INFO  (TEST-LeaderElectionContextKeyTest.test-seed#[352ADDAB357047D5]) [    ] o.a.s.SolrTestCaseJ4 ###Starting test
   [junit4]   2> 1992111 INFO  (qtp896082360-20407) [n:127.0.0.1:61523_solr    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.testCollection1.shard2.replica_n3, tag=null
   [junit4]   2> 1992111 INFO  (qtp896082360-20407) [n:127.0.0.1:61523_solr    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@2f856946: rootName = solr_61523, domain = solr.core.testCollection1.shard2.replica_n3, service url = null, agent id = null] for registry solr.core.testCollection1.shard2.replica_n3 / com.codahale.metrics.MetricRegistry@192c64fc
   [junit4]   2> 1992126 INFO  (qtp896082360-20407) [n:127.0.0.1:61523_solr    ] o.a.s.c.SolrCore [testCollection1_shard2_replica_n3]  CLOSING SolrCore org.apache.solr.core.SolrCore@3484bb01
   [junit4]   2> 1992126 INFO  (qtp896082360-20407) [n:127.0.0.1:61523_solr    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.testCollection1.shard2.replica_n3, tag=881113857
   [junit4]   2> 1992126 INFO  (qtp896082360-20407) [n:127.0.0.1:61523_solr    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.testCollection1.shard2.leader, tag=881113857
   [junit4]   2> 1992139 INFO  (qtp896082360-20407) [n:127.0.0.1:61523_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={deleteInstanceDir=false&core=testCollection1_shard2_replica_n3&deleteDataDir=false&action=UNLOAD&indexInfo=true&wt=javabin&version=2&deleteIndex=false} status=0 QTime=28
   [junit4]   2> 1992140 INFO  (TEST-LeaderElectionContextKeyTest.test-seed#[352ADDAB357047D5]) [    ] o.a.s.SolrTestCaseJ4 ###Ending test
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=LeaderElectionContextKeyTest -Dtests.method=test -Dtests.seed=352ADDAB357047D5 -Dtests.slow=true -Dtests.locale=lv -Dtests.timezone=Africa/Johannesburg -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] FAILURE 0.19s J1 | LeaderElectionContextKeyTest.test <<<
   [junit4]    > Throwable #1: java.lang.AssertionError
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([352ADDAB357047D5:BD7EE2719B8C2A2D]:0)
   [junit4]    > 	at org.apache.solr.cloud.LeaderElectionContextKeyTest.test(LeaderElectionContextKeyTest.java:88)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1992146 INFO  (jetty-closer-3576-thread-1) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@391438f3{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 1992149 INFO  (jetty-closer-3576-thread-1) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1348109751
   [junit4]   2> 1992149 INFO  (jetty-closer-3576-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 1992149 INFO  (jetty-closer-3576-thread-1) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@51034df4: rootName = solr_61523, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@6e2f9026
   [junit4]   2> 1992153 INFO  (jetty-closer-3576-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 1992153 INFO  (jetty-closer-3576-thread-1) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@3b0739af: rootName = solr_61523, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@41a480e9
   [junit4]   2> 1992155 INFO  (jetty-closer-3576-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 1992156 INFO  (jetty-closer-3576-thread-1) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@15241ac7: rootName = solr_61523, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@6abf3e4b
   [junit4]   2> 1992157 INFO  (jetty-closer-3576-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
   [junit4]   2> 1992160 INFO  (coreCloseExecutor-8575-thread-2) [n:127.0.0.1:61523_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.c.SolrCore [testCollection2_shard2_replica_n2]  CLOSING SolrCore org.apache.solr.core.SolrCore@9a27c78
   [junit4]   2> 1992160 INFO  (coreCloseExecutor-8575-thread-2) [n:127.0.0.1:61523_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.testCollection2.shard2.replica_n2, tag=161643640
   [junit4]   2> 1992160 INFO  (coreCloseExecutor-8575-thread-2) [n:127.0.0.1:61523_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@1b82dcab: rootName = solr_61523, domain = solr.core.testCollection2.shard2.replica_n2, service url = null, agent id = null] for registry solr.core.testCollection2.shard2.replica_n2 / com.codahale.metrics.MetricRegistry@68c24b4
   [junit4]   2> 1992160 INFO  (coreCloseExecutor-8575-thread-3) [n:127.0.0.1:61523_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.c.SolrCore [testCollection2_shard1_replica_n1]  CLOSING SolrCore org.apache.solr.core.SolrCore@3ec0ffe6
   [junit4]   2> 1992163 INFO  (coreCloseExecutor-8575-thread-1) [n:127.0.0.1:61523_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.c.SolrCore [testCollection1_shard1_replica_n1]  CLOSING SolrCore org.apache.solr.core.SolrCore@4e571b1c
   [junit4]   2> 1992170 INFO  (coreCloseExecutor-8575-thread-2) [n:127.0.0.1:61523_solr c:testCollection2 s:shard2 r:core_node4 x:testCollection2_shard2_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.testCollection2.shard2.leader, tag=161643640
   [junit4]   2> 1992170 INFO  (coreCloseExecutor-8575-thread-3) [n:127.0.0.1:61523_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.testCollection2.shard1.replica_n1, tag=1052835814
   [junit4]   2> 1992170 INFO  (coreCloseExecutor-8575-thread-3) [n:127.0.0.1:61523_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@4cb22622: rootName = solr_61523, domain = solr.core.testCollection2.shard1.replica_n1, service url = null, agent id = null] for registry solr.core.testCollection2.shard1.replica_n1 / com.codahale.metrics.MetricRegistry@56737e0a
   [junit4]   2> 1992176 INFO  (coreCloseExecutor-8575-thread-3) [n:127.0.0.1:61523_solr c:testCollection2 s:shard1 r:core_node3 x:testCollection2_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.testCollection2.shard1.leader, tag=1052835814
   [junit4]   2> 1992176 INFO  (coreCloseExecutor-8575-thread-1) [n:127.0.0.1:61523_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.testCollection1.shard1.replica_n1, tag=1314331420
   [junit4]   2> 1992176 INFO  (coreCloseExecutor-8575-thread-1) [n:127.0.0.1:61523_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@7eda3645: rootName = solr_61523, domain = solr.core.testCollection1.shard1.replica_n1, service url = null, agent id = null] for registry solr.core.testCollection1.shard1.replica_n1 / com.codahale.metrics.MetricRegistry@4a879169
   [junit4]   2> 1992183 INFO  (coreCloseExecutor-8575-thread-1) [n:127.0.0.1:61523_solr c:testCollection1 s:shard1 r:core_node2 x:testCollection1_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.testCollection1.shard1.leader, tag=1314331420
   [junit4]   2> 1992184 INFO  (jetty-closer-3576-thread-1) [    ] o.a.s.c.Overseer Overseer (id=98578534662995971-127.0.0.1:61523_solr-n_0000000000) closing
   [junit4]   2> 1992184 INFO  (OverseerStateUpdate-98578534662995971-127.0.0.1:61523_solr-n_0000000000) [n:127.0.0.1:61523_solr    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:61523_solr
   [junit4]   2> 1992185 WARN  (zkCallback-3582-thread-1-processing-n:127.0.0.1:61523_solr) [n:127.0.0.1:61523_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 1992186 INFO  (jetty-closer-3576-thread-1) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@6f8c08aa{/solr,null,UNAVAILABLE}
   [junit4]   2> 1992187 ERROR (SUITE-LeaderElectionContextKeyTest-seed#[352ADDAB357047D5]-worker) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 1992187 INFO  (SUITE-LeaderElectionContextKeyTest-seed#[352ADDAB357047D5]-worker) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:61521 61521
   [junit4]   2> 1992206 INFO  (Thread-3318) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:61521 61521
   [junit4]   2> 1992453 WARN  (Thread-3318) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	2	/solr/aliases.json
   [junit4]   2> 	2	/solr/clusterprops.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	9	/solr/collections/testCollection1/state.json
   [junit4]   2> 	5	/solr/collections/testCollection2/state.json
   [junit4]   2> 	2	/solr/clusterstate.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	2	/solr/live_nodes
   [junit4]   2> 	2	/solr/collections
   [junit4]   2> 
   [junit4]   2> NOTE: leaving temporary files on disk at: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderElectionContextKeyTest_352ADDAB357047D5-001
   [junit4]   2> Aug 31, 2017 2:17:53 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=FastCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=FAST, chunkSize=4, maxDocsPerChunk=754, blockSize=759), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=FAST, chunkSize=4, blockSize=759)), sim=RandomSimilarity(queryNorm=false): {}, locale=lv, timezone=Africa/Johannesburg
   [junit4]   2> NOTE: Mac OS X 10.11.6 x86_64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=3,threads=1,free=266385896,total=536870912
   [junit4]   2> NOTE: All tests run in this JVM: [TestComponentsName, RecoveryAfterSoftCommitTest, HdfsThreadLeakTest, TestReload, AddBlockUpdateTest, TestPostingsSolrHighlighter, PKIAuthenticationIntegrationTest, DeleteLastCustomShardedReplicaTest, CustomCollectionTest, ShardRoutingCustomTest, TestHashQParserPlugin, CheckHdfsIndexTest, DeleteStatusTest, CoreAdminCreateDiscoverTest, TestHalfAndHalfDocValues, TestCloudSchemaless, StatsComponentTest, TestEmbeddedSolrServerSchemaAPI, TestConfigSetsAPI, OutOfBoxZkACLAndCredentialsProvidersTest, DocValuesMultiTest, TestUnifiedSolrHighlighter, OpenExchangeRatesOrgProviderTest, TestBlendedInfixSuggestions, DirectoryFactoryTest, ImplicitSnitchTest, TestClassicSimilarityFactory, SuggesterFSTTest, TestManagedResourceStorage, ConnectionReuseTest, TestCoreAdminApis, CollectionsAPIDistributedZkTest, TestConfigSetImmutable, StressHdfsTest, TestCrossCoreJoin, TestCollationField, MoveReplicaHDFSUlogDirTest, TestSerializedLuceneMatchVersion, TestReplicationHandlerBackup, TestCloudDeleteByQuery, TestCustomStream, TestHdfsUpdateLog, TestSolrCoreSnapshots, TestBulkSchemaConcurrent, TestRemoteStreaming, LeaderElectionIntegrationTest, TestSearchPerf, TestCSVLoader, TestCoreDiscovery, MissingSegmentRecoveryTest, TestDeleteCollectionOnDownNodes, RulesTest, ClassificationUpdateProcessorTest, BooleanFieldTest, TestStressCloudBlindAtomicUpdates, TestDocumentBuilder, CloudExitableDirectoryReaderTest, DocExpirationUpdateProcessorFactoryTest, DocumentAnalysisRequestHandlerTest, LukeRequestHandlerTest, DistributedQueryComponentOptimizationTest, OverseerTaskQueueTest, TestAnalyzedSuggestions, ShardSplitTest, PolyFieldTest, TestStressLiveNodes, TestDistribIDF, TestSimpleTextCodec, BasicZkTest, TestPKIAuthenticationPlugin, BadComponentTest, VersionInfoTest, MetricsHandlerTest, TestFastOutputStream, ExternalFileFieldSortTest, TestFreeTextSuggestions, TestRecovery, TestSolrConfigHandlerConcurrent, SharedFSAutoReplicaFailoverUtilsTest, TestLegacyFieldCache, TestSmileRequest, TestMaxTokenLenTokenizer, TestMacros, VMParamsZkACLAndCredentialsProvidersTest, TestLMJelinekMercerSimilarityFactory, TestJoin, TestSystemIdResolver, TestZkAclsWithHadoopAuth, TestSolrCloudWithKerberosAlt, SolrCoreCheckLockOnStartupTest, TestEmbeddedSolrServerAdminHandler, SpellPossibilityIteratorTest, TestMergePolicyConfig, TestNumericTokenStream, ZkStateReaderTest, FacetPivotSmallTest, TestStressLucene, CdcrUpdateLogTest, TestSchemalessBufferedUpdates, JvmMetricsTest, BlockJoinFacetRandomTest, TestPointFields, TestSchemaSimilarityResource, DistribDocExpirationUpdateProcessorTest, OverseerCollectionConfigSetProcessorTest, HdfsChaosMonkeyNothingIsSafeTest, HdfsLockFactoryTest, TestElisionMultitermQuery, TestNonDefinedSimilarityFactory, TestNoOpRegenerator, TestNumericTerms64, TestImpersonationWithHadoopAuth, TestManagedSchemaAPI, ZkNodePropsTest, LeaderFailureAfterFreshStartTest, HLLSerializationTest, UninvertDocValuesMergePolicyTest, TestRestoreCore, DistributedIntervalFacetingTest, TestDynamicFieldCollectionResource, UUIDUpdateProcessorFallbackTest, DistributedExpandComponentTest, TestPayloadCheckQParserPlugin, BadIndexSchemaTest, CurrencyFieldTypeTest, DocValuesNotIndexedTest, TestExpandComponent, ExplicitHLLTest, SOLR749Test, TestStreamBody, TestSchemaNameResource, CdcrRequestHandlerTest, ConnectionManagerTest, DistributedMLTComponentTest, TestFastWriter, SpellCheckCollatorWithCollapseTest, TestCustomDocTransformer, FieldMutatingUpdateProcessorTest, SolrXmlInZkTest, HdfsBasicDistributedZk2Test, TestJavabinTupleStreamParser, TestFieldCacheVsDocValues, HdfsDirectoryFactoryTest, TestComplexPhraseLeadingWildcard, SpellCheckCollatorTest, TestFilteredDocIdSet, SystemInfoHandlerTest, TestBackupRepositoryFactory, LargeFieldTest, TestCustomSort, JavabinLoaderTest, SolrCoreMetricManagerTest, SecurityConfHandlerTest, GraphQueryTest, ZkSolrClientTest, TestGeoJSONResponseWriter, FileUtilsTest, BufferStoreTest, TestFieldTypeCollectionResource, TestInitParams, SpellCheckComponentTest, SolrCmdDistributorTest, ResponseHeaderTest, TestSQLHandlerNonCloud, AtomicUpdateProcessorFactoryTest, SolrGangliaReporterTest, HdfsRestartWhileUpdatingTest, SampleTest, DirectSolrSpellCheckerTest, TestTolerantUpdateProcessorCloud, AutoCommitTest, BasicFunctionalityTest, SolrRequestParserTest, TestReloadAndDeleteDocs, TestSolrQueryParser, TestSolrCoreParser, SuggesterTSTTest, TestWordDelimiterFilterFactory, TestRecoveryHdfs, BasicDistributedZkTest, ChaosMonkeySafeLeaderTest, BasicDistributedZk2Test, UnloadDistributedZkTest, OpenCloseCoreStressTest, OverseerTest, FullSolrCloudDistribCmdsTest, TestZkChroot, TestDistributedGrouping, TestHashPartitioner, TermVectorComponentDistributedTest, TestStressReorder, TestStressVersions, DistributedTermsComponentTest, TestSolr4Spatial, QueryElevationComponentTest, TestLazyCores, ShowFileRequestHandlerTest, SolrIndexSplitterTest, SignatureUpdateProcessorFactoryTest, TestExtendedDismaxParser, CoreAdminHandlerTest, SuggesterTest, TestTrie, NoCacheHeaderTest, SchemaVersionSpecificBehaviorTest, TestPseudoReturnFields, TestUpdate, StatelessScriptUpdateProcessorFactoryTest, DistanceFunctionTest, XsltUpdateRequestHandlerTest, CacheHeaderTest, PrimitiveFieldTypeTest, TestOmitPositions, RequestHandlersTest, DocumentBuilderTest, TestValueSourceCache, PathHierarchyTokenizerFactoryTest, TermVectorComponentTest, IndexSchemaRuntimeFieldTest, RegexBoostProcessorTest, TestJmxIntegration, MBeansHandlerTest, HighlighterConfigTest, UpdateParamsTest, TestSolrIndexConfig, CopyFieldTest, ResponseLogComponentTest, SolrIndexConfigTest, TestDocSet, MinimalSchemaTest, TestFuzzyAnalyzedSuggestions, NotRequiredUniqueKeyTest, TestCharFilters, SynonymTokenizerTest, EchoParamsTest, TestPerFieldSimilarity, TestBM25SimilarityFactory, URLClassifyProcessorTest, TestFastLRUCache, PreAnalyzedFieldTest, SpellingQueryConverterTest, RAMDirectoryFactoryTest, TestUtils, CircularListTest, TestRTGBase, SolrTestCaseJ4Test, TestCursorMarkWithoutUniqueKey, TestDistributedMissingSort, TestTolerantSearch, AliasIntegrationTest, AssignTest, CdcrReplicationDistributedZkTest, CdcrVersionReplicationTest, ChaosMonkeyNothingIsSafeWithPullReplicasTest, CollectionStateFormat2Test, CollectionsAPIAsyncDistributedZkTest, DeleteShardTest, DistributedVersionInfoTest, LeaderElectionContextKeyTest]
   [junit4] Completed [521/732 (1!)] on J1 in 7.62s, 1 test, 1 failure <<< FAILURES!

[...truncated 161 lines...]
   [junit4] Suite: org.apache.solr.cloud.ZkFailoverTest
   [junit4]   2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ZkFailoverTest_352ADDAB357047D5-001/init-core-data-001
   [junit4]   2> 2527250 INFO  (SUITE-ZkFailoverTest-seed#[352ADDAB357047D5]-worker) [    ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 2527256 INFO  (SUITE-ZkFailoverTest-seed#[352ADDAB357047D5]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN) w/ MAC_OS_X supressed clientAuth
   [junit4]   2> 2527256 INFO  (SUITE-ZkFailoverTest-seed#[352ADDAB357047D5]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 2527258 INFO  (SUITE-ZkFailoverTest-seed#[352ADDAB357047D5]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ZkFailoverTest_352ADDAB357047D5-001/tempDir-001
   [junit4]   2> 2527258 INFO  (SUITE-ZkFailoverTest-seed#[352ADDAB357047D5]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 2527258 INFO  (Thread-3909) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 2527262 INFO  (Thread-3909) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 2527266 ERROR (Thread-3909) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 2527366 INFO  (SUITE-ZkFailoverTest-seed#[352ADDAB357047D5]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:63632
   [junit4]   2> 2527422 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15e38aff2ba0000, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 2527431 INFO  (jetty-launcher-4101-thread-1) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 2527434 INFO  (jetty-launcher-4101-thread-2) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 2527444 INFO  (jetty-launcher-4101-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@16fdbb58{/solr,null,AVAILABLE}
   [junit4]   2> 2527444 INFO  (jetty-launcher-4101-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4f6ee6f{/solr,null,AVAILABLE}
   [junit4]   2> 2527444 INFO  (jetty-launcher-4101-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@6588d83d{HTTP/1.1,[http/1.1]}{127.0.0.1:63634}
   [junit4]   2> 2527444 INFO  (jetty-launcher-4101-thread-2) [    ] o.e.j.s.Server Started @2531679ms
   [junit4]   2> 2527445 INFO  (jetty-launcher-4101-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=63634}
   [junit4]   2> 2527446 INFO  (jetty-launcher-4101-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@491daffe{HTTP/1.1,[http/1.1]}{127.0.0.1:63635}
   [junit4]   2> 2527446 ERROR (jetty-launcher-4101-thread-2) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2527446 INFO  (jetty-launcher-4101-thread-1) [    ] o.e.j.s.Server Started @2531681ms
   [junit4]   2> 2527446 INFO  (jetty-launcher-4101-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=63635}
   [junit4]   2> 2527446 INFO  (jetty-launcher-4101-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 2527446 ERROR (jetty-launcher-4101-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2527446 INFO  (jetty-launcher-4101-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2527446 INFO  (jetty-launcher-4101-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 2527446 INFO  (jetty-launcher-4101-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-08-31T14:26:48.815Z
   [junit4]   2> 2527446 INFO  (jetty-launcher-4101-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 2527446 INFO  (jetty-launcher-4101-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2527446 INFO  (jetty-launcher-4101-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 2527446 INFO  (jetty-launcher-4101-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-08-31T14:26:48.815Z
   [junit4]   2> 2527511 INFO  (jetty-launcher-4101-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 2527515 INFO  (jetty-launcher-4101-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 2527538 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15e38aff2ba0002, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 2527557 INFO  (jetty-launcher-4101-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:63632/solr
   [junit4]   2> 2527558 INFO  (jetty-launcher-4101-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:63632/solr
   [junit4]   2> 2527948 INFO  (jetty-launcher-4101-thread-1) [n:127.0.0.1:63635_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2527949 INFO  (jetty-launcher-4101-thread-1) [n:127.0.0.1:63635_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:63635_solr
   [junit4]   2> 2527968 INFO  (jetty-launcher-4101-thread-1) [n:127.0.0.1:63635_solr    ] o.a.s.c.Overseer Overseer (id=98578570191437830-127.0.0.1:63635_solr-n_0000000000) starting
   [junit4]   2> 2527985 INFO  (jetty-launcher-4101-thread-1) [n:127.0.0.1:63635_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:63635_solr
   [junit4]   2> 2527988 INFO  (zkCallback-4113-thread-1-processing-n:127.0.0.1:63635_solr) [n:127.0.0.1:63635_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2528003 INFO  (jetty-launcher-4101-thread-2) [n:127.0.0.1:63634_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2528007 INFO  (jetty-launcher-4101-thread-2) [n:127.0.0.1:63634_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2528011 INFO  (jetty-launcher-4101-thread-2) [n:127.0.0.1:63634_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:63634_solr
   [junit4]   2> 2528012 INFO  (zkCallback-4113-thread-1-processing-n:127.0.0.1:63635_solr) [n:127.0.0.1:63635_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2528013 INFO  (zkCallback-4112-thread-1-processing-n:127.0.0.1:63634_solr) [n:127.0.0.1:63634_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2528269 INFO  (jetty-launcher-4101-thread-2) [n:127.0.0.1:63634_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_63634.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7da5c4cc
   [junit4]   2> 2528269 INFO  (jetty-launcher-4101-thread-1) [n:127.0.0.1:63635_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_63635.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7da5c4cc
   [junit4]   2> 2528280 INFO  (jetty-launcher-4101-thread-1) [n:127.0.0.1:63635_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_63635.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7da5c4cc
   [junit4]   2> 2528280 INFO  (jetty-launcher-4101-thread-2) [n:127.0.0.1:63634_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_63634.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7da5c4cc
   [junit4]   2> 2528280 INFO  (jetty-launcher-4101-thread-1) [n:127.0.0.1:63635_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_63635.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7da5c4cc
   [junit4]   2> 2528280 INFO  (jetty-launcher-4101-thread-2) [n:127.0.0.1:63634_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_63634.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7da5c4cc
   [junit4]   2> 2528283 INFO  (jetty-launcher-4101-thread-2) [n:127.0.0.1:63634_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ZkFailoverTest_352ADDAB357047D5-001/tempDir-001/node2/.
   [junit4]   2> 2528283 INFO  (jetty-launcher-4101-thread-1) [n:127.0.0.1:63635_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ZkFailoverTest_352ADDAB357047D5-001/tempDir-001/node1/.
   [junit4]   2> 2528346 INFO  (SUITE-ZkFailoverTest-seed#[352ADDAB357047D5]-worker) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2528351 INFO  (SUITE-ZkFailoverTest-seed#[352ADDAB357047D5]-worker) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:63632/solr ready
   [junit4]   2> 2528487 INFO  (TEST-ZkFailoverTest.testRestartZkWhenClusterDown-seed#[352ADDAB357047D5]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testRestartZkWhenClusterDown
   [junit4]   2> 2528498 INFO  (qtp914262124-23921) [n:127.0.0.1:63635_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params pullReplicas=0&replicationFactor=1&name=coll1&nrtReplicas=1&action=CREATE&numShards=2&tlogReplicas=0&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2528501 INFO  (OverseerThreadFactory-10024-thread-1-processing-n:127.0.0.1:63635_solr) [n:127.0.0.1:63635_solr    ] o.a.s.c.CreateCollectionCmd Create collection coll1
   [junit4]   2> 2528502 INFO  (OverseerThreadFactory-10024-thread-1-processing-n:127.0.0.1:63635_solr) [n:127.0.0.1:63635_solr    ] o.a.s.c.CreateCollectionCmd Only _default config set found, using it.
   [junit4]   2> 2528503 INFO  (OverseerThreadFactory-10024-thread-1-processing-n:127.0.0.1:63635_solr) [n:127.0.0.1:63635_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/managed-schema to /configs/coll1/managed-schema
   [junit4]   2> 2528507 INFO  (OverseerThreadFactory-10024-thread-1-processing-n:127.0.0.1:63635_solr) [n:127.0.0.1:63635_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/protwords.txt to /configs/coll1/protwords.txt
   [junit4]   2> 2528516 INFO  (OverseerThreadFactory-10024-thread-1-processing-n:127.0.0.1:63635_solr) [n:127.0.0.1:63635_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/solrconfig.xml to /configs/coll1/solrconfig.xml
   [junit4]   2> 2528520 INFO  (OverseerThreadFactory-10024-thread-1-processing-n:127.0.0.1:63635_solr) [n:127.0.0.1:63635_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/synonyms.txt to /configs/coll1/synonyms.txt
   [junit4]   2> 2528523 INFO  (OverseerThreadFactory-10024-thread-1-processing-n:127.0.0.1:63635_solr) [n:127.0.0.1:63635_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/stopwords.txt to /configs/coll1/stopwords.txt
   [junit4]   2> 2528526 INFO  (OverseerThreadFactory-10024-thread-1-processing-n:127.0.0.1:63635_solr) [n:127.0.0.1:63635_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/contractions_it.txt to /configs/coll1/lang/contractions_it.txt
   [junit4]   2> 2528534 INFO  (OverseerThreadFactory-10024-thread-1-processing-n:127.0.0.1:63635_solr) [n:127.0.0.1:63635_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stemdict_nl.txt to /configs/coll1/lang/stemdict_nl.txt
   [junit4]   2> 2528537 INFO  (OverseerThreadFactory-10024-thread-1-processing-n:127.0.0.1:63635_solr) [n:127.0.0.1:63635_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_hy.txt to /configs/coll1/lang/stopwords_hy.txt
   [junit4]   2> 2528545 INFO  (OverseerThreadFactory-10024-thread-1-processing-n:127.0.0.1:63635_solr) [n:127.0.0.1:63635_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_no.txt to /configs/coll1/lang/stopwords_no.txt
   [junit4]   2> 2528551 INFO  (OverseerThreadFactory-10024-thread-1-processing-n:127.0.0.1:63635_solr) [n:127.0.0.1:63635_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_id.txt to /configs/coll1/lang/stopwords_id.txt
   [junit4]   2> 2528555 INFO  (OverseerThreadFactory-10024-thread-1-processing-n:127.0.0.1:63635_solr) [n:127.0.0.1:63635_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_da.txt to /configs/coll1/lang/stopwords_da.txt
   [junit4]   2> 2528557 INFO  (OverseerThreadFactory-10024-thread-1-processing-n:127.0.0.1:63635_solr) [n:127.0.0.1:63635_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_ga.txt to /configs/coll1/lang/stopwords_ga.txt
   [junit4]   2> 2528560 INFO  (OverseerThreadFactory-10024-thread-1-processing-n:127.0.0.1:63635_solr) [n:127.0.0.1:63635_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_hi.txt to /configs/coll1/lang/stopwords_hi.txt
   [junit4]   2> 2528563 INFO  (OverseerThreadFactory-10024-thread-1-processing-n:127.0.0.1:63635_solr) [n:127.0.0.1:63635_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_ja.txt to /configs/coll1/lang/stopwords_ja.txt
   [junit4]   2> 2528566 INFO  (OverseerThreadFactory-10024-thread-1-processing-n:127.0.0.1:63635_solr) [n:127.0.0.1:63635_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_pt.txt to /configs/coll1/lang/stopwords_pt.txt
   [junit4]   2> 2528569 INFO  (OverseerThreadFactory-10024-thread-1-processing-n:127.0.0.1:63635_solr) [n:127.0.0.1:63635_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_hu.txt to /configs/coll1/lang/stopwords_hu.txt
   [junit4]   2> 2528577 INFO  (OverseerThreadFactory-10024-thread-1-processing-n:127.0.0.1:63635_solr) [n:127.0.0.1:63635_solr    ] o.a.s.c.c.ZkConfigManager Copying zk node /configs/_default/lang/stopwords_el.txt to /confi

[...truncated too long message...]

VAILABLE}
   [junit4]   2> 2563117 WARN  (zkCallback-4113-thread-1-processing-n:127.0.0.1:63635_solr) [n:127.0.0.1:63635_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 2563118 INFO  (jetty-closer-4102-thread-1) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@4f6ee6f{/solr,null,UNAVAILABLE}
   [junit4]   2> 2563129 ERROR (SUITE-ZkFailoverTest-seed#[352ADDAB357047D5]-worker) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 2563129 INFO  (SUITE-ZkFailoverTest-seed#[352ADDAB357047D5]-worker) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:63632 63632
   [junit4]   2> 2563151 INFO  (Thread-3909) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:63632 63632
   [junit4]   2> 2564386 WARN  (Thread-3909) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	3	/solr/aliases.json
   [junit4]   2> 	3	/solr/clusterprops.json
   [junit4]   2> 	2	/solr/security.json
   [junit4]   2> 	2	/solr/collections/coll1/state.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	6	/solr/collections/coll1/state.json
   [junit4]   2> 	3	/solr/clusterstate.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	3	/solr/live_nodes
   [junit4]   2> 	3	/solr/collections
   [junit4]   2> 
   [junit4]   2> NOTE: leaving temporary files on disk at: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ZkFailoverTest_352ADDAB357047D5-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=1810, maxMBSortInHeap=7.4487596001320755, sim=RandomSimilarity(queryNorm=true): {}, locale=fr, timezone=Asia/Thimphu
   [junit4]   2> NOTE: Mac OS X 10.11.6 x86_64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=3,threads=1,free=180774624,total=536870912
   [junit4]   2> NOTE: All tests run in this JVM: [TestComponentsName, RecoveryAfterSoftCommitTest, HdfsThreadLeakTest, TestReload, AddBlockUpdateTest, TestPostingsSolrHighlighter, PKIAuthenticationIntegrationTest, DeleteLastCustomShardedReplicaTest, CustomCollectionTest, ShardRoutingCustomTest, TestHashQParserPlugin, CheckHdfsIndexTest, DeleteStatusTest, CoreAdminCreateDiscoverTest, TestHalfAndHalfDocValues, TestCloudSchemaless, StatsComponentTest, TestEmbeddedSolrServerSchemaAPI, TestConfigSetsAPI, OutOfBoxZkACLAndCredentialsProvidersTest, DocValuesMultiTest, TestUnifiedSolrHighlighter, OpenExchangeRatesOrgProviderTest, TestBlendedInfixSuggestions, DirectoryFactoryTest, ImplicitSnitchTest, TestClassicSimilarityFactory, SuggesterFSTTest, TestManagedResourceStorage, ConnectionReuseTest, TestCoreAdminApis, CollectionsAPIDistributedZkTest, TestConfigSetImmutable, StressHdfsTest, TestCrossCoreJoin, TestCollationField, MoveReplicaHDFSUlogDirTest, TestSerializedLuceneMatchVersion, TestReplicationHandlerBackup, TestCloudDeleteByQuery, TestCustomStream, TestHdfsUpdateLog, TestSolrCoreSnapshots, TestBulkSchemaConcurrent, TestRemoteStreaming, LeaderElectionIntegrationTest, TestSearchPerf, TestCSVLoader, TestCoreDiscovery, MissingSegmentRecoveryTest, TestDeleteCollectionOnDownNodes, RulesTest, ClassificationUpdateProcessorTest, BooleanFieldTest, TestStressCloudBlindAtomicUpdates, TestDocumentBuilder, CloudExitableDirectoryReaderTest, DocExpirationUpdateProcessorFactoryTest, DocumentAnalysisRequestHandlerTest, LukeRequestHandlerTest, DistributedQueryComponentOptimizationTest, OverseerTaskQueueTest, TestAnalyzedSuggestions, ShardSplitTest, PolyFieldTest, TestStressLiveNodes, TestDistribIDF, TestSimpleTextCodec, BasicZkTest, TestPKIAuthenticationPlugin, BadComponentTest, VersionInfoTest, MetricsHandlerTest, TestFastOutputStream, ExternalFileFieldSortTest, TestFreeTextSuggestions, TestRecovery, TestSolrConfigHandlerConcurrent, SharedFSAutoReplicaFailoverUtilsTest, TestLegacyFieldCache, TestSmileRequest, TestMaxTokenLenTokenizer, TestMacros, VMParamsZkACLAndCredentialsProvidersTest, TestLMJelinekMercerSimilarityFactory, TestJoin, TestSystemIdResolver, TestZkAclsWithHadoopAuth, TestSolrCloudWithKerberosAlt, SolrCoreCheckLockOnStartupTest, TestEmbeddedSolrServerAdminHandler, SpellPossibilityIteratorTest, TestMergePolicyConfig, TestNumericTokenStream, ZkStateReaderTest, FacetPivotSmallTest, TestStressLucene, CdcrUpdateLogTest, TestSchemalessBufferedUpdates, JvmMetricsTest, BlockJoinFacetRandomTest, TestPointFields, TestSchemaSimilarityResource, DistribDocExpirationUpdateProcessorTest, OverseerCollectionConfigSetProcessorTest, HdfsChaosMonkeyNothingIsSafeTest, HdfsLockFactoryTest, TestElisionMultitermQuery, TestNonDefinedSimilarityFactory, TestNoOpRegenerator, TestNumericTerms64, TestImpersonationWithHadoopAuth, TestManagedSchemaAPI, ZkNodePropsTest, LeaderFailureAfterFreshStartTest, HLLSerializationTest, UninvertDocValuesMergePolicyTest, TestRestoreCore, DistributedIntervalFacetingTest, TestDynamicFieldCollectionResource, UUIDUpdateProcessorFallbackTest, DistributedExpandComponentTest, TestPayloadCheckQParserPlugin, BadIndexSchemaTest, CurrencyFieldTypeTest, DocValuesNotIndexedTest, TestExpandComponent, ExplicitHLLTest, SOLR749Test, TestStreamBody, TestSchemaNameResource, CdcrRequestHandlerTest, ConnectionManagerTest, DistributedMLTComponentTest, TestFastWriter, SpellCheckCollatorWithCollapseTest, TestCustomDocTransformer, FieldMutatingUpdateProcessorTest, SolrXmlInZkTest, HdfsBasicDistributedZk2Test, TestJavabinTupleStreamParser, TestFieldCacheVsDocValues, HdfsDirectoryFactoryTest, TestComplexPhraseLeadingWildcard, SpellCheckCollatorTest, TestFilteredDocIdSet, SystemInfoHandlerTest, TestBackupRepositoryFactory, LargeFieldTest, TestCustomSort, JavabinLoaderTest, SolrCoreMetricManagerTest, SecurityConfHandlerTest, GraphQueryTest, ZkSolrClientTest, TestGeoJSONResponseWriter, FileUtilsTest, BufferStoreTest, TestFieldTypeCollectionResource, TestInitParams, SpellCheckComponentTest, SolrCmdDistributorTest, ResponseHeaderTest, TestSQLHandlerNonCloud, AtomicUpdateProcessorFactoryTest, SolrGangliaReporterTest, HdfsRestartWhileUpdatingTest, SampleTest, DirectSolrSpellCheckerTest, TestTolerantUpdateProcessorCloud, AutoCommitTest, BasicFunctionalityTest, SolrRequestParserTest, TestReloadAndDeleteDocs, TestSolrQueryParser, TestSolrCoreParser, SuggesterTSTTest, TestWordDelimiterFilterFactory, TestRecoveryHdfs, BasicDistributedZkTest, ChaosMonkeySafeLeaderTest, BasicDistributedZk2Test, UnloadDistributedZkTest, OpenCloseCoreStressTest, OverseerTest, FullSolrCloudDistribCmdsTest, TestZkChroot, TestDistributedGrouping, TestHashPartitioner, TermVectorComponentDistributedTest, TestStressReorder, TestStressVersions, DistributedTermsComponentTest, TestSolr4Spatial, QueryElevationComponentTest, TestLazyCores, ShowFileRequestHandlerTest, SolrIndexSplitterTest, SignatureUpdateProcessorFactoryTest, TestExtendedDismaxParser, CoreAdminHandlerTest, SuggesterTest, TestTrie, NoCacheHeaderTest, SchemaVersionSpecificBehaviorTest, TestPseudoReturnFields, TestUpdate, StatelessScriptUpdateProcessorFactoryTest, DistanceFunctionTest, XsltUpdateRequestHandlerTest, CacheHeaderTest, PrimitiveFieldTypeTest, TestOmitPositions, RequestHandlersTest, DocumentBuilderTest, TestValueSourceCache, PathHierarchyTokenizerFactoryTest, TermVectorComponentTest, IndexSchemaRuntimeFieldTest, RegexBoostProcessorTest, TestJmxIntegration, MBeansHandlerTest, HighlighterConfigTest, UpdateParamsTest, TestSolrIndexConfig, CopyFieldTest, ResponseLogComponentTest, SolrIndexConfigTest, TestDocSet, MinimalSchemaTest, TestFuzzyAnalyzedSuggestions, NotRequiredUniqueKeyTest, TestCharFilters, SynonymTokenizerTest, EchoParamsTest, TestPerFieldSimilarity, TestBM25SimilarityFactory, URLClassifyProcessorTest, TestFastLRUCache, PreAnalyzedFieldTest, SpellingQueryConverterTest, RAMDirectoryFactoryTest, TestUtils, CircularListTest, TestRTGBase, SolrTestCaseJ4Test, TestCursorMarkWithoutUniqueKey, TestDistributedMissingSort, TestTolerantSearch, AliasIntegrationTest, AssignTest, CdcrReplicationDistributedZkTest, CdcrVersionReplicationTest, ChaosMonkeyNothingIsSafeWithPullReplicasTest, CollectionStateFormat2Test, CollectionsAPIAsyncDistributedZkTest, DeleteShardTest, DistributedVersionInfoTest, LeaderElectionContextKeyTest, LeaderFailoverAfterPartitionTest, LeaderInitiatedRecoveryOnShardRestartTest, MoveReplicaHDFSTest, MultiThreadedOCPTest, SSLMigrationTest, SimpleCollectionCreateDeleteTest, TestCloudInspectUtil, TestCloudJSONFacetJoinDomain, TestCloudPseudoReturnFields, TestCloudSearcherWarming, TestDistribDocBasedVersion, TestHdfsCloudBackupRestore, TestLeaderElectionWithEmptyReplica, TestLocalFSCloudBackupRestore, TestOnReconnectListenerSupport, TestPullReplica, TestRandomFlRTGCloud, TestRandomRequestDistribution, TestReplicaProperties, TestRequestStatusCollectionAPI, TestSizeLimitedDistributedMap, TlogReplayBufferedWhileIndexingTest, ZkFailoverTest]
   [junit4] Completed [571/732 (2!)] on J1 in 37.17s, 1 test, 1 error <<< FAILURES!

[...truncated 46659 lines...]
BUILD FAILED
/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/build.xml:810: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/build.xml:690: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/build.xml:678: Source checkout is modified!!! Offending files:
* solr/licenses/httpmime-4.5.3.jar.sha1

Total time: 87 minutes 42 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any

[JENKINS] Lucene-Solr-master-MacOSX (64bit/jdk1.8.0) - Build # 4158 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-MacOSX/4158/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseParallelGC

174 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.AnalysisAfterCoreReloadTest

Error Message:
org.apache.solr.AnalysisAfterCoreReloadTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.AnalysisAfterCoreReloadTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/AnalysisAfterCoreReloadTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.BasicFunctionalityTest

Error Message:
org.apache.solr.BasicFunctionalityTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.BasicFunctionalityTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/BasicFunctionalityTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.OutputWriterTest

Error Message:
org.apache.solr.OutputWriterTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.OutputWriterTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/OutputWriterTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.SampleTest

Error Message:
org.apache.solr.SampleTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.SampleTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/SampleTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.SolrInfoBeanTest

Error Message:
org.apache.solr.SolrInfoBeanTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.SolrInfoBeanTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/SolrInfoBeanTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.TestCrossCoreJoin

Error Message:
org.apache.solr.TestCrossCoreJoin

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.TestCrossCoreJoin
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/TestCrossCoreJoin.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.TestHighlightDedupGrouping

Error Message:
org.apache.solr.TestHighlightDedupGrouping

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.TestHighlightDedupGrouping
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/TestHighlightDedupGrouping.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.analysis.TestReversedWildcardFilterFactory

Error Message:
org.apache.solr.analysis.TestReversedWildcardFilterFactory

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.analysis.TestReversedWildcardFilterFactory
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/analysis/TestReversedWildcardFilterFactory.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.analysis.TestWordDelimiterFilterFactory

Error Message:
org.apache.solr.analysis.TestWordDelimiterFilterFactory

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.analysis.TestWordDelimiterFilterFactory
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/analysis/TestWordDelimiterFilterFactory.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.client.solrj.embedded.TestEmbeddedSolrServerAdminHandler

Error Message:
org.apache.solr.client.solrj.embedded.TestEmbeddedSolrServerAdminHandler

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.client.solrj.embedded.TestEmbeddedSolrServerAdminHandler
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/client/solrj/embedded/TestEmbeddedSolrServerAdminHandler.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.client.solrj.embedded.TestEmbeddedSolrServerConstructors

Error Message:
org.apache.solr.client.solrj.embedded.TestEmbeddedSolrServerConstructors

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.client.solrj.embedded.TestEmbeddedSolrServerConstructors
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/client/solrj/embedded/TestEmbeddedSolrServerConstructors.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.CdcrReplicationHandlerTest

Error Message:
org.apache.solr.cloud.CdcrReplicationHandlerTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.cloud.CdcrReplicationHandlerTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/cloud/CdcrReplicationHandlerTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.CleanupOldIndexTest

Error Message:
org.apache.solr.cloud.CleanupOldIndexTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.cloud.CleanupOldIndexTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/cloud/CleanupOldIndexTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.CollectionReloadTest

Error Message:
org.apache.solr.cloud.CollectionReloadTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.cloud.CollectionReloadTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/cloud/CollectionReloadTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.CollectionStateFormat2Test

Error Message:
org.apache.solr.cloud.CollectionStateFormat2Test

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.cloud.CollectionStateFormat2Test
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/cloud/CollectionStateFormat2Test.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.CollectionsAPISolrJTest

Error Message:
org.apache.solr.cloud.CollectionsAPISolrJTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.cloud.CollectionsAPISolrJTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/cloud/CollectionsAPISolrJTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest

Error Message:
org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/cloud/ConcurrentDeleteAndCreateCollectionTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ConfigSetsAPITest

Error Message:
org.apache.solr.cloud.ConfigSetsAPITest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.cloud.ConfigSetsAPITest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/cloud/ConfigSetsAPITest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.DeleteLastCustomShardedReplicaTest

Error Message:
org.apache.solr.cloud.DeleteLastCustomShardedReplicaTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.cloud.DeleteLastCustomShardedReplicaTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/cloud/DeleteLastCustomShardedReplicaTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.DeleteNodeTest

Error Message:
org.apache.solr.cloud.DeleteNodeTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.cloud.DeleteNodeTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/cloud/DeleteNodeTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.DeleteReplicaTest

Error Message:
org.apache.solr.cloud.DeleteReplicaTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.cloud.DeleteReplicaTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/cloud/DeleteReplicaTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.DeleteShardTest

Error Message:
org.apache.solr.cloud.DeleteShardTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.cloud.DeleteShardTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/cloud/DeleteShardTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.DeleteStatusTest

Error Message:
org.apache.solr.cloud.DeleteStatusTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.cloud.DeleteStatusTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/cloud/DeleteStatusTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.DistribCursorPagingTest

Error Message:
org.apache.solr.cloud.DistribCursorPagingTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.cloud.DistribCursorPagingTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/cloud/DistribCursorPagingTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  org.apache.solr.cloud.DistributedQueueTest.testLocallyOffer

Error Message:
Could not get the port for ZooKeeper server

Stack Trace:
java.lang.RuntimeException: Could not get the port for ZooKeeper server
	at org.apache.solr.cloud.ZkTestServer.run(ZkTestServer.java:515)
	at org.apache.solr.cloud.DistributedQueueTest.setupZk(DistributedQueueTest.java:332)
	at org.apache.solr.cloud.DistributedQueueTest.setUp(DistributedQueueTest.java:47)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:941)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.DistributedQueueTest.testDistributedQueue

Error Message:
Could not get the port for ZooKeeper server

Stack Trace:
java.lang.RuntimeException: Could not get the port for ZooKeeper server
	at org.apache.solr.cloud.ZkTestServer.run(ZkTestServer.java:515)
	at org.apache.solr.cloud.DistributedQueueTest.setupZk(DistributedQueueTest.java:332)
	at org.apache.solr.cloud.DistributedQueueTest.setUp(DistributedQueueTest.java:47)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:941)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.DistributedQueueTest.testPeekElements

Error Message:
Could not get the port for ZooKeeper server

Stack Trace:
java.lang.RuntimeException: Could not get the port for ZooKeeper server
	at org.apache.solr.cloud.ZkTestServer.run(ZkTestServer.java:515)
	at org.apache.solr.cloud.DistributedQueueTest.setupZk(DistributedQueueTest.java:332)
	at org.apache.solr.cloud.DistributedQueueTest.setUp(DistributedQueueTest.java:47)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:941)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.DistributedQueueTest.testDistributedQueueCache

Error Message:
Could not get the port for ZooKeeper server

Stack Trace:
java.lang.RuntimeException: Could not get the port for ZooKeeper server
	at org.apache.solr.cloud.ZkTestServer.run(ZkTestServer.java:515)
	at org.apache.solr.cloud.DistributedQueueTest.setupZk(DistributedQueueTest.java:332)
	at org.apache.solr.cloud.DistributedQueueTest.setUp(DistributedQueueTest.java:47)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:941)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.DistributedQueueTest.testLeakChildWatcher

Error Message:
Could not get the port for ZooKeeper server

Stack Trace:
java.lang.RuntimeException: Could not get the port for ZooKeeper server
	at org.apache.solr.cloud.ZkTestServer.run(ZkTestServer.java:515)
	at org.apache.solr.cloud.DistributedQueueTest.setupZk(DistributedQueueTest.java:332)
	at org.apache.solr.cloud.DistributedQueueTest.setUp(DistributedQueueTest.java:47)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:941)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.DistributedQueueTest.testDistributedQueueBlocking

Error Message:
Could not get the port for ZooKeeper server

Stack Trace:
java.lang.RuntimeException: Could not get the port for ZooKeeper server
	at org.apache.solr.cloud.ZkTestServer.run(ZkTestServer.java:515)
	at org.apache.solr.cloud.DistributedQueueTest.setupZk(DistributedQueueTest.java:332)
	at org.apache.solr.cloud.DistributedQueueTest.setUp(DistributedQueueTest.java:47)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:941)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.DistributedVersionInfoTest

Error Message:
org.apache.solr.cloud.DistributedVersionInfoTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.cloud.DistributedVersionInfoTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/cloud/DistributedVersionInfoTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.DocValuesNotIndexedTest

Error Message:
org.apache.solr.cloud.DocValuesNotIndexedTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.cloud.DocValuesNotIndexedTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/cloud/DocValuesNotIndexedTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ForceLeaderTest

Error Message:
org.apache.solr.cloud.ForceLeaderTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.cloud.ForceLeaderTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/cloud/ForceLeaderTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.HealthCheckHandlerTest

Error Message:
org.apache.solr.cloud.HealthCheckHandlerTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.cloud.HealthCheckHandlerTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/cloud/HealthCheckHandlerTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.HttpPartitionTest

Error Message:
org.apache.solr.cloud.HttpPartitionTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.cloud.HttpPartitionTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/cloud/HttpPartitionTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.LeaderElectionContextKeyTest

Error Message:
org.apache.solr.cloud.LeaderElectionContextKeyTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.cloud.LeaderElectionContextKeyTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/cloud/LeaderElectionContextKeyTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.LeaderFailoverAfterPartitionTest

Error Message:
org.apache.solr.cloud.LeaderFailoverAfterPartitionTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.cloud.LeaderFailoverAfterPartitionTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/cloud/LeaderFailoverAfterPartitionTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest

Error Message:
org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/cloud/LeaderInitiatedRecoveryOnCommitTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.MigrateRouteKeyTest

Error Message:
org.apache.solr.cloud.MigrateRouteKeyTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.cloud.MigrateRouteKeyTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/cloud/MigrateRouteKeyTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.MoveReplicaHDFSUlogDirTest

Error Message:
org.apache.solr.cloud.MoveReplicaHDFSUlogDirTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.cloud.MoveReplicaHDFSUlogDirTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/cloud/MoveReplicaHDFSUlogDirTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.NodeMutatorTest

Error Message:
org.apache.solr.cloud.NodeMutatorTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.cloud.NodeMutatorTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java.security.AccessController.doPrivileged(Native Method)
	at java.net.URLClassLoader.findClass(URLClassLoader.java:361)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
	at java.lang.Class.forName0(Native Method)
	at java.lang.Class.forName(Class.java:348)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.instantiate(SlaveMain.java:280)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:240)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:368)
	at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
Caused by: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/classes/test/org/apache/solr/cloud/NodeMutatorTest.class (Too many open files)
	at java.io.FileInputStream.open0(Native Method)
	at java.io.FileInputStream.open(FileInputStream.java:195)
	at java.io.FileInputStream.<init>(FileInputStream.java:138)
	at sun.misc.URLClassPath$FileLoader$1.getInputStream(URLClassPath.java:1288)
	at sun.misc.Resource.cachedInputStream(Resource.java:77)
	at sun.misc.Resource.getByteBuffer(Resource.java:160)
	at java.net.URLClassLoader.defineClass(URLClassLoader.java:454)
	at java.net.URLClassLoader.access$100(URLClassLoader.java:73)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:368)
	... 12 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.OverseerCollectionConfigSetProcessorTest

Error Message:
org.apache.solr.cloud.OverseerCollectionConfigSetProcessorTest

Stack Trace:
java.lang.ClassNotFoundException: org.apache.solr.cloud.OverseerCollectionConfigSetProcessorTest
	at java.net.URLClassLoader$1.run(URLClassLoader.java:370)
	at java.net.URLClassLoader$1.run(URLClassLoader.java:362)
	at java

[...truncated too long message...]

60)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >   10) Thread[id=17959, name=zkCallback-3072-thread-3, state=TIMED_WAITING, group=TGRP-RecoveryAfterSoftCommitTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >   11) Thread[id=17953, name=OverseerStateUpdate-98575958299901956-127.0.0.1:62699_-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]    >         at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:165)
   [junit4]    >         at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:139)
   [junit4]    >         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:184)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >   12) Thread[id=17958, name=zkCallback-3072-thread-2, state=TIMED_WAITING, group=TGRP-RecoveryAfterSoftCommitTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([274D5BE395376E31]:0)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   [junit4]    >    1) Thread[id=17950, name=TEST-RecoveryAfterSoftCommitTest.test-seed#[274D5BE395376E31]-SendThread(127.0.0.1:62690), state=TIMED_WAITING, group=TGRP-RecoveryAfterSoftCommitTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)
   [junit4]    >    2) Thread[id=17952, name=zkCallback-3072-thread-1, state=TIMED_WAITING, group=TGRP-RecoveryAfterSoftCommitTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    3) Thread[id=17960, name=zkCallback-3072-thread-4, state=TIMED_WAITING, group=TGRP-RecoveryAfterSoftCommitTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    4) Thread[id=17961, name=zkCallback-3072-thread-5, state=TIMED_WAITING, group=TGRP-RecoveryAfterSoftCommitTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    5) Thread[id=17959, name=zkCallback-3072-thread-3, state=TIMED_WAITING, group=TGRP-RecoveryAfterSoftCommitTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    6) Thread[id=17958, name=zkCallback-3072-thread-2, state=TIMED_WAITING, group=TGRP-RecoveryAfterSoftCommitTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([274D5BE395376E31]:0)
   [junit4] Completed [506/732 (144!)] on J1 in 27.31s, 1 test, 3 errors <<< FAILURES!

[...truncated 46866 lines...]
BUILD FAILED
/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/build.xml:810: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/build.xml:690: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/build.xml:678: Source checkout is modified!!! Offending files:
* solr/licenses/httpmime-4.5.3.jar.sha1

Total time: 77 minutes 30 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any

[JENKINS] Lucene-Solr-master-MacOSX (64bit/jdk1.8.0) - Build # 4157 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-MacOSX/4157/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseParallelGC

5 tests failed.
FAILED:  org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.test

Error Message:
Could not find collection:collection2

Stack Trace:
java.lang.AssertionError: Could not find collection:collection2
	at __randomizedtesting.SeedInfo.seed([BF615684E0E1E696:3735695E4E1D8B6E]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNotNull(Assert.java:526)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:155)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:140)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:135)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:908)
	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.testIndexingBatchPerRequestWithHttpSolrClient(FullSolrCloudDistribCmdsTest.java:612)
	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.test(FullSolrCloudDistribCmdsTest.java:152)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.LeaderFailureAfterFreshStartTest.test

Error Message:
timeout waiting to see all nodes active

Stack Trace:
java.lang.AssertionError: timeout waiting to see all nodes active
	at __randomizedtesting.SeedInfo.seed([BF615684E0E1E696:3735695E4E1D8B6E]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.LeaderFailureAfterFreshStartTest.waitTillNodesActive(LeaderFailureAfterFreshStartTest.java:233)
	at org.apache.solr.cloud.LeaderFailureAfterFreshStartTest.restartNodes(LeaderFailureAfterFreshStartTest.java:173)
	at org.apache.solr.cloud.LeaderFailureAfterFreshStartTest.test(LeaderFailureAfterFreshStartTest.java:148)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.LeaderFailureAfterFreshStartTest

Error Message:
12 threads leaked from SUITE scope at org.apache.solr.cloud.LeaderFailureAfterFreshStartTest:     1) Thread[id=16811, name=Connection evictor, state=TIMED_WAITING, group=TGRP-LeaderFailureAfterFreshStartTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=16831, name=zkCallback-3123-thread-2, state=TIMED_WAITING, group=TGRP-LeaderFailureAfterFreshStartTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=16850, name=zkCallback-3123-thread-6, state=TIMED_WAITING, group=TGRP-LeaderFailureAfterFreshStartTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    4) Thread[id=16841, name=zkCallback-3123-thread-3, state=TIMED_WAITING, group=TGRP-LeaderFailureAfterFreshStartTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    5) Thread[id=16812, name=Connection evictor, state=TIMED_WAITING, group=TGRP-LeaderFailureAfterFreshStartTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)    6) Thread[id=16816, name=TEST-LeaderFailureAfterFreshStartTest.test-seed#[BF615684E0E1E696]-SendThread(127.0.0.1:61957), state=TIMED_WAITING, group=TGRP-LeaderFailureAfterFreshStartTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)    7) Thread[id=16849, name=zkCallback-3123-thread-5, state=TIMED_WAITING, group=TGRP-LeaderFailureAfterFreshStartTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    8) Thread[id=16842, name=OverseerStateUpdate-98573432764956693-127.0.0.1:61983_-n_0000000005, state=TIMED_WAITING, group=Overseer state updater.]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:165)         at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:139)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:184)         at java.lang.Thread.run(Thread.java:748)    9) Thread[id=16848, name=zkCallback-3123-thread-4, state=TIMED_WAITING, group=TGRP-LeaderFailureAfterFreshStartTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   10) Thread[id=16843, name=OverseerCollectionConfigSetProcessor-98573432764956693-127.0.0.1:61983_-n_0000000005, state=TIMED_WAITING, group=Overseer collection creation process.]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354)         at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:387)         at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:193)         at java.lang.Thread.run(Thread.java:748)   11) Thread[id=16817, name=TEST-LeaderFailureAfterFreshStartTest.test-seed#[BF615684E0E1E696]-EventThread, state=WAITING, group=TGRP-LeaderFailureAfterFreshStartTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)   12) Thread[id=16844, name=OverseerHdfsCoreFailoverThread-98573432764956693-127.0.0.1:61983_-n_0000000005, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:139)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 12 threads leaked from SUITE scope at org.apache.solr.cloud.LeaderFailureAfterFreshStartTest: 
   1) Thread[id=16811, name=Connection evictor, state=TIMED_WAITING, group=TGRP-LeaderFailureAfterFreshStartTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=16831, name=zkCallback-3123-thread-2, state=TIMED_WAITING, group=TGRP-LeaderFailureAfterFreshStartTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=16850, name=zkCallback-3123-thread-6, state=TIMED_WAITING, group=TGRP-LeaderFailureAfterFreshStartTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   4) Thread[id=16841, name=zkCallback-3123-thread-3, state=TIMED_WAITING, group=TGRP-LeaderFailureAfterFreshStartTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   5) Thread[id=16812, name=Connection evictor, state=TIMED_WAITING, group=TGRP-LeaderFailureAfterFreshStartTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:748)
   6) Thread[id=16816, name=TEST-LeaderFailureAfterFreshStartTest.test-seed#[BF615684E0E1E696]-SendThread(127.0.0.1:61957), state=TIMED_WAITING, group=TGRP-LeaderFailureAfterFreshStartTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
   7) Thread[id=16849, name=zkCallback-3123-thread-5, state=TIMED_WAITING, group=TGRP-LeaderFailureAfterFreshStartTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   8) Thread[id=16842, name=OverseerStateUpdate-98573432764956693-127.0.0.1:61983_-n_0000000005, state=TIMED_WAITING, group=Overseer state updater.]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:165)
        at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:139)
        at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:184)
        at java.lang.Thread.run(Thread.java:748)
   9) Thread[id=16848, name=zkCallback-3123-thread-4, state=TIMED_WAITING, group=TGRP-LeaderFailureAfterFreshStartTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  10) Thread[id=16843, name=OverseerCollectionConfigSetProcessor-98573432764956693-127.0.0.1:61983_-n_0000000005, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:354)
        at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:387)
        at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:193)
        at java.lang.Thread.run(Thread.java:748)
  11) Thread[id=16817, name=TEST-LeaderFailureAfterFreshStartTest.test-seed#[BF615684E0E1E696]-EventThread, state=WAITING, group=TGRP-LeaderFailureAfterFreshStartTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
  12) Thread[id=16844, name=OverseerHdfsCoreFailoverThread-98573432764956693-127.0.0.1:61983_-n_0000000005, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:139)
        at java.lang.Thread.run(Thread.java:748)
	at __randomizedtesting.SeedInfo.seed([BF615684E0E1E696]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.LeaderFailureAfterFreshStartTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=16816, name=TEST-LeaderFailureAfterFreshStartTest.test-seed#[BF615684E0E1E696]-SendThread(127.0.0.1:61957), state=TIMED_WAITING, group=TGRP-LeaderFailureAfterFreshStartTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)    2) Thread[id=16831, name=zkCallback-3123-thread-2, state=TIMED_WAITING, group=TGRP-LeaderFailureAfterFreshStartTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=16849, name=zkCallback-3123-thread-5, state=TIMED_WAITING, group=TGRP-LeaderFailureAfterFreshStartTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    4) Thread[id=16850, name=zkCallback-3123-thread-6, state=TIMED_WAITING, group=TGRP-LeaderFailureAfterFreshStartTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    5) Thread[id=16841, name=zkCallback-3123-thread-3, state=TIMED_WAITING, group=TGRP-LeaderFailureAfterFreshStartTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    6) Thread[id=16848, name=zkCallback-3123-thread-4, state=TIMED_WAITING, group=TGRP-LeaderFailureAfterFreshStartTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=16816, name=TEST-LeaderFailureAfterFreshStartTest.test-seed#[BF615684E0E1E696]-SendThread(127.0.0.1:61957), state=TIMED_WAITING, group=TGRP-LeaderFailureAfterFreshStartTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
   2) Thread[id=16831, name=zkCallback-3123-thread-2, state=TIMED_WAITING, group=TGRP-LeaderFailureAfterFreshStartTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=16849, name=zkCallback-3123-thread-5, state=TIMED_WAITING, group=TGRP-LeaderFailureAfterFreshStartTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   4) Thread[id=16850, name=zkCallback-3123-thread-6, state=TIMED_WAITING, group=TGRP-LeaderFailureAfterFreshStartTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   5) Thread[id=16841, name=zkCallback-3123-thread-3, state=TIMED_WAITING, group=TGRP-LeaderFailureAfterFreshStartTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   6) Thread[id=16848, name=zkCallback-3123-thread-4, state=TIMED_WAITING, group=TGRP-LeaderFailureAfterFreshStartTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
	at __randomizedtesting.SeedInfo.seed([BF615684E0E1E696]:0)


FAILED:  org.apache.solr.cloud.MultiThreadedOCPTest.test

Error Message:
Task 3001 did not complete, final state: FAILED expected same:<COMPLETED> was not:<FAILED>

Stack Trace:
java.lang.AssertionError: Task 3001 did not complete, final state: FAILED expected same:<COMPLETED> was not:<FAILED>
	at __randomizedtesting.SeedInfo.seed([BF615684E0E1E696:3735695E4E1D8B6E]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotSame(Assert.java:641)
	at org.junit.Assert.assertSame(Assert.java:580)
	at org.apache.solr.cloud.MultiThreadedOCPTest.testDeduplicationOfSubmittedTasks(MultiThreadedOCPTest.java:226)
	at org.apache.solr.cloud.MultiThreadedOCPTest.test(MultiThreadedOCPTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 11737 lines...]
   [junit4] Suite: org.apache.solr.cloud.FullSolrCloudDistribCmdsTest
   [junit4]   2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_BF615684E0E1E696-001/init-core-data-001
   [junit4]   2> 722882 WARN  (SUITE-FullSolrCloudDistribCmdsTest-seed#[BF615684E0E1E696]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=9 numCloses=9
   [junit4]   2> 722892 ERROR (Thread-1423) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 723745 ERROR (TEST-FullSolrCloudDistribCmdsTest.test-seed#[BF615684E0E1E696]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 725389 WARN  (qtp1971369419-6743) [n:127.0.0.1:58313_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 726640 WARN  (OverseerThreadFactory-3023-thread-2-processing-n:127.0.0.1:58313_) [n:127.0.0.1:58313_    ] o.a.s.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
   [junit4]   2> 727226 ERROR (TEST-FullSolrCloudDistribCmdsTest.test-seed#[BF615684E0E1E696]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 728608 WARN  (qtp475229596-6783) [n:127.0.0.1:58328_ c:collection1 s:shard2  x:collection1_shard2_replica_n61] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 729301 ERROR (TEST-FullSolrCloudDistribCmdsTest.test-seed#[BF615684E0E1E696]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 730946 WARN  (qtp1194345807-6814) [n:127.0.0.1:58340_ c:collection1 s:shard3  x:collection1_shard3_replica_n63] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 731622 ERROR (TEST-FullSolrCloudDistribCmdsTest.test-seed#[BF615684E0E1E696]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 733793 WARN  (qtp1404806737-6841) [n:127.0.0.1:58348_ c:collection1 s:shard1  x:collection1_shard1_replica_n65] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 734357 ERROR (TEST-FullSolrCloudDistribCmdsTest.test-seed#[BF615684E0E1E696]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 735709 WARN  (qtp1484574634-6870) [n:127.0.0.1:58363_ c:collection1 s:shard2  x:collection1_shard2_replica_n67] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 736082 ERROR (TEST-FullSolrCloudDistribCmdsTest.test-seed#[BF615684E0E1E696]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 737403 WARN  (qtp712235149-6902) [n:127.0.0.1:58369_ c:collection1 s:shard3  x:collection1_shard3_replica_n69] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 737858 ERROR (TEST-FullSolrCloudDistribCmdsTest.test-seed#[BF615684E0E1E696]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 739246 WARN  (qtp1095894781-6935) [n:127.0.0.1:58376_ c:collection1 s:shard1  x:collection1_shard1_replica_n71] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 744157 WARN  (zkCallback-1243-thread-1-processing-n:127.0.0.1:58348_) [n:127.0.0.1:58348_    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 744174 WARN  (zkCallback-1263-thread-2-processing-n:127.0.0.1:58376_) [n:127.0.0.1:58376_    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 745161 WARN  (zkCallback-1263-thread-1-processing-n:127.0.0.1:58376_) [n:127.0.0.1:58376_ c:collection1 s:shard1 r:core_node72 x:collection1_shard1_replica_n71] o.a.s.c.SyncStrategy Closed, skipping sync up.
   [junit4]   2> 750236 ERROR (qtp475229596-6780) [n:127.0.0.1:58328_ c:collection1 s:shard2 r:core_node62 x:collection1_shard2_replica_n61] o.a.s.h.RequestHandlerBase org.apache.solr.common.SolrException: No registered leader was found after waiting for 4000ms , collection: collection1 slice: shard1
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:758)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:744)
   [junit4]   2> 	at org.apache.solr.update.processor.DistributedUpdateProcessor.setupRequest(DistributedUpdateProcessor.java:404)
   [junit4]   2> 	at org.apache.solr.update.processor.DistributedUpdateProcessor.setupRequest(DistributedUpdateProcessor.java:350)
   [junit4]   2> 	at org.apache.solr.update.processor.DistributedUpdateProcessor.processAdd(DistributedUpdateProcessor.java:708)
   [junit4]   2> 	at org.apache.solr.handler.loader.JavabinLoader$1.update(JavabinLoader.java:98)
   [junit4]   2> 	at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readOuterMostDocIterator(JavaBinUpdateRequestCodec.java:188)
   [junit4]   2> 	at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readIterator(JavaBinUpdateRequestCodec.java:144)
   [junit4]   2> 	at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:311)
   [junit4]   2> 	at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:256)
   [junit4]   2> 	at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readNamedList(JavaBinUpdateRequestCodec.java:130)
   [junit4]   2> 	at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:276)
   [junit4]   2> 	at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:256)
   [junit4]   2> 	at org.apache.solr.common.util.JavaBinCodec.unmarshal(JavaBinCodec.java:178)
   [junit4]   2> 	at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.unmarshal(JavaBinUpdateRequestCodec.java:195)
   [junit4]   2> 	at org.apache.solr.handler.loader.JavabinLoader.parseAndLoadDocs(JavabinLoader.java:108)
   [junit4]   2> 	at org.apache.solr.handler.loader.JavabinLoader.load(JavabinLoader.java:55)
   [junit4]   2> 	at org.apache.solr.handler.UpdateRequestHandler$1.load(UpdateRequestHandler.java:97)
   [junit4]   2> 	at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:68)
   [junit4]   2> 	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.execute(SolrCore.java:2483)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:720)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:526)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 
   [junit4]   2> 750382 WARN  (zkCallback-1249-thread-1-processing-n:127.0.0.1:58363_) [n:127.0.0.1:58363_    ] o.a.s.c.LeaderElector 
   [junit4]   2> org.apache.solr.common.SolrException: SolrCore not found:collection1_shard2_replica_n67 in []
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:297)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:56)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:348)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$3.lambda$process$0(SolrZkClient.java:269)
   [junit4]   2> 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 750386 WARN  (zkCallback-1256-thread-1-processing-n:127.0.0.1:58369_) [n:127.0.0.1:58369_    ] o.a.s.c.LeaderElector 
   [junit4]   2> org.apache.solr.common.SolrException: SolrCore not found:collection1_shard3_replica_n69 in []
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:297)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:56)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:348)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$3.lambda$process$0(SolrZkClient.java:269)
   [junit4]   2> 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 753028 WARN  (qtp712235149-6904) [n:127.0.0.1:58369_ c:collection2 s:shard1  x:collection2_shard1_replica_n3] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 753070 WARN  (qtp475229596-6785) [n:127.0.0.1:58328_ c:collection2 s:shard1  x:collection2_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 753080 WARN  (qtp1971369419-6746) [n:127.0.0.1:58313_ c:collection2 s:shard1  x:collection2_shard1_replica_n2] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 753184 WARN  (qtp475229596-6786) [n:127.0.0.1:58328_ c:collection2 s:shard2  x:collection2_shard2_replica_n8] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 753205 WARN  (qtp1484574634-6875) [n:127.0.0.1:58363_ c:collection2 s:shard2  x:collection2_shard2_replica_n5] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 753207 WARN  (qtp475229596-6783) [n:127.0.0.1:58328_ c:collection2 s:shard4  x:collection2_shard4_replica_n15] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 753264 WARN  (qtp1971369419-6741) [n:127.0.0.1:58313_ c:collection2 s:shard6  x:collection2_shard6_replica_n24] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 753273 WARN  (qtp1194345807-6816) [n:127.0.0.1:58340_ c:collection2 s:shard2  x:collection2_shard2_replica_n6] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 753276 WARN  (qtp712235149-6903) [n:127.0.0.1:58369_ c:collection2 s:shard3  x:collection2_shard3_replica_n11] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 753277 WARN  (qtp712235149-6905) [n:127.0.0.1:58369_ c:collection2 s:shard5  x:collection2_shard5_replica_n18] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 753387 WARN  (qtp1971369419-6743) [n:127.0.0.1:58313_ c:collection2 s:shard3  x:collection2_shard3_replica_n9] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 753397 WARN  (qtp475229596-6780) [n:127.0.0.1:58328_ c:collection2 s:shard7  x:collection2_shard7_replica_n30] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 753397 WARN  (qtp475229596-6782) [n:127.0.0.1:58328_ c:collection2 s:shard6  x:collection2_shard6_replica_n23] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 753397 WARN  (qtp1194345807-6815) [n:127.0.0.1:58340_ c:collection2 s:shard5  x:collection2_shard5_replica_n21] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 753400 WARN  (qtp1484574634-6870) [n:127.0.0.1:58363_ c:collection2 s:shard3  x:collection2_shard3_replica_n12] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 753418 WARN  (qtp1971369419-6742) [n:127.0.0.1:58313_ c:collection2 s:shard4  x:collection2_shard4_replica_n17] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 753418 WARN  (qtp1194345807-6817) [n:127.0.0.1:58340_ c:collection2 s:shard4  x:collection2_shard4_replica_n14] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 753423 WARN  (qtp1484574634-6868) [n:127.0.0.1:58363_ c:collection2 s:shard7  x:collection2_shard7_replica_n27] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 753430 WARN  (qtp1484574634-6871) [n:127.0.0.1:58363_ c:collection2 s:shard5  x:collection2_shard5_replica_n20] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 753430 WARN  (qtp712235149-6906) [n:127.0.0.1:58369_ c:collection2 s:shard6  x:collection2_shard6_replica_n26] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 753461 WARN  (qtp1194345807-6813) [n:127.0.0.1:58340_ c:collection2 s:shard7  x:collection2_shard7_replica_n29] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 753673 ERROR (qtp1194345807-6813) [n:127.0.0.1:58340_ c:collection2 s:shard7  x:collection2_shard7_replica_n29] o.a.s.h.RequestHandlerBase org.apache.solr.common.SolrException: Error CREATEing SolrCore 'collection2_shard7_replica_n29': Unable to create core [collection2_shard7_replica_n29] Caused by: invalid API spec: apispec/core.config.Commands.json
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:971)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)
   [junit4]   2> 	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:745)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:726)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:507)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: Unable to create core [collection2_shard7_replica_n29]
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1018)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:938)
   [junit4]   2> 	... 34 more
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: invalid API spec: apispec/core.config.Commands.json
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:988)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:843)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1002)
   [junit4]   2> 	... 35 more
   [junit4]   2> Caused by: java.lang.RuntimeException: invalid API spec: apispec/core.config.Commands.json
   [junit4]   2> 	at org.apache.solr.common.util.ValidatingJsonMap.parse(ValidatingJsonMap.java:318)
   [junit4]   2> 	at org.apache.solr.common.util.Utils.lambda$getSpec$0(Utils.java:394)
   [junit4]   2> 	at org.apache.solr.api.Api.getSpec(Api.java:65)
   [junit4]   2> 	at org.apache.solr.api.ApiBag.register(ApiBag.java:73)
   [junit4]   2> 	at org.apache.solr.core.PluginBag.put(PluginBag.java:217)
   [junit4]   2> 	at org.apache.solr.core.PluginBag.init(PluginBag.java:274)
   [junit4]   2> 	at org.apache.solr.core.RequestHandlers.initHandlersFromConfig(RequestHandlers.java:130)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:946)
   [junit4]   2> 	... 37 more
   [junit4]   2> 
   [junit4]   2> 753674 ERROR (OverseerThreadFactory-3023-thread-5-processing-n:127.0.0.1:58313_) [n:127.0.0.1:58313_    ] o.a.s.c.OverseerCollectionMessageHandler Error from shard: http://127.0.0.1:58340
   [junit4]   2> org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:58340: Error CREATEing SolrCore 'collection2_shard7_replica_n29': Unable to create core [collection2_shard7_replica_n29] Caused by: invalid API spec: apispec/core.config.Commands.json
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:627)
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
   [junit4]   2> 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
   [junit4]   2> 	at org.apache.solr.handler.component.HttpShardHandler.lambda$submit$0(HttpShardHandler.java:172)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 753678 ERROR (qtp1484574634-6871) [n:127.0.0.1:58363_ c:collection2 s:shard5  x:collection2_shard5_replica_n20] o.a.s.h.RequestHandlerBase org.apache.solr.common.SolrException: Error CREATEing SolrCore 'collection2_shard5_replica_n20': Unable to create core [collection2_shard5_replica_n20] Caused by: invalid API spec: apispec/core.SchemaEdit.json
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:971)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)
   [junit4]   2> 	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:745)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:726)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:507)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: Unable to create core [collection2_shard5_replica_n20]
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1018)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:938)
   [junit4]   2> 	... 34 more
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: invalid API spec: apispec/core.SchemaEdit.json
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:988)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:843)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1002)
   [junit4]   2> 	... 35 more
   [junit4]   2> Caused by: java.lang.RuntimeException: invalid API spec: apispec/core.SchemaEdit.json
   [junit4]   2> 	at org.apache.solr.common.util.ValidatingJsonMap.parse(ValidatingJsonMap.java:318)
   [junit4]   2> 	at org.apache.solr.common.util.Utils.lambda$getSpec$0(Utils.java:394)
   [junit4]   2> 	at org.apache.solr.api.Api.getSpec(Api.java:65)
   [junit4]   2> 	at org.apache.solr.api.ApiBag.register(ApiBag.java:73)
   [junit4]   2> 	at org.apache.solr.core.PluginBag.put(PluginBag.java:217)
   [junit4]   2> 	at org.apache.solr.core.PluginBag.init(PluginBag.java:274)
   [junit4]   2> 	at org.apache.solr.core.RequestHandlers.initHandlersFromConfig(RequestHandlers.java:130)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:946)
   [junit4]   2> 	... 37 more
   [junit4]   2> 
   [junit4]   2> 753679 ERROR (OverseerThreadFactory-3023-thread-5-processing-n:127.0.0.1:58313_) [n:127.0.0.1:58313_    ] o.a.s.c.OverseerCollectionMessageHandler Error from shard: http://127.0.0.1:58363
   [junit4]   2> org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:58363: Error CREATEing SolrCore 'collection2_shard5_replica_n20': Unable to create core [collection2_shard5_replica_n20] Caused by: invalid API spec: apispec/core.SchemaEdit.json
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:627)
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
   [junit4]   2> 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
   [junit4]   2> 	at org.apache.solr.handler.component.HttpShardHandler.lambda$submit$0(HttpShardHandler.java:172)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 753680 ERROR (qtp712235149-6906) [n:127.0.0.1:58369_ c:collection2 s:shard6  x:collection2_shard6_replica_n26] o.a.s.h.RequestHandlerBase org.apache.solr.common.SolrException: Error CREATEing SolrCore 'collection2_shard6_replica_n26': Unable to create core [collection2_shard6_replica_n26] Caused by: invalid API spec: apispec/core.SchemaEdit.json
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:971)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)
   [junit4]   2> 	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:745)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:726)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:507)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: Unable to create core [collection2_shard6_replica_n26]
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1018)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:938)
   [junit4]   2> 	... 34 more
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: invalid API spec: apispec/core.SchemaEdit.json
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:988)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:843)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1002)
   [junit4]   2> 	... 35 more
   [junit4]   2> Caused by: java.lang.RuntimeException: invalid API spec: apispec/core.SchemaEdit.json
   [junit4]   2> 	at org.apache.solr.common.util.ValidatingJsonMap.parse(ValidatingJsonMap.java:318)
   [junit4]   2> 	at org.apache.solr.common.util.Utils.lambda$getSpec$0(Utils.java:394)
   [junit4]   2> 	at org.apache.solr.api.Api.getSpec(Api.java:65)
   [junit4]   2> 	at org.apache.solr.api.ApiBag.register(ApiBag.java:73)
   [junit4]   2> 	at org.apache.solr.core.PluginBag.put(PluginBag.java:217)
   [junit4]   2> 	at org.apache.solr.core.PluginBag.init(PluginBag.java:274)
   [junit4]   2> 	at org.apache.solr.core.RequestHandlers.initHandlersFromConfig(RequestHandlers.java:130)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:946)
   [junit4]   2> 	... 37 more
   [junit4]   2> 
   [junit4]   2> 753681 ERROR (OverseerThreadFactory-3023-thread-5-processing-n:127.0.0.1:58313_) [n:127.0.0.1:58313_    ] o.a.s.c.OverseerCollectionMessageHandler Error from shard: http://127.0.0.1:58369
   [junit4]   2> org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:58369: Error CREATEing SolrCore 'collection2_shard6_replica_n26': Unable to create core [collection2_shard6_replica_n26] Caused by: invalid API spec: apispec/core.SchemaEdit.json
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:627)
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
   [junit4]   2> 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
   [junit4]   2> 	at org.apache.solr.handler.component.HttpShardHandler.lambda$submit$0(HttpShardHandler.java:172)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 753683 ERROR (qtp1971369419-6742) [n:127.0.0.1:58313_ c:collection2 s:shard4  x:collection2_shard4_replica_n17] o.a.s.h.RequestHandlerBase org.apache.solr.common.SolrException: Error CREATEing SolrCore 'collection2_shard4_replica_n17': Unable to create core [collection2_shard4_replica_n17] Caused by: invalid API spec: apispec/core.SchemaEdit.json
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:971)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)
   [junit4]   2> 	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:745)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:726)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:507)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: Unable to create core [collection2_shard4_replica_n17]
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1018)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:938)
   [junit4]   2> 	... 34 more
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: invalid API spec: apispec/core.SchemaEdit.json
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:988)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:843)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1002)
   [junit4]   2> 	... 35 more
   [junit4]   2> Caused by: java.lang.RuntimeException: invalid API spec: apispec/core.SchemaEdit.json
   [junit4]   2> 	at org.apache.solr.common.util.ValidatingJsonMap.parse(ValidatingJsonMap.java:318)
   [junit4]   2> 	at org.apache.solr.common.util.Utils.lambda$getSpec$0(Utils.java:394)
   [junit4]   2> 	at org.apache.solr.api.Api.getSpec(Api.java:65)
   [junit4]   2> 	at org.apache.solr.api.ApiBag.register(ApiBag.java:73)
   [junit4]   2> 	at org.apache.solr.core.PluginBag.put(PluginBag.java:217)
   [junit4]   2> 	at org.apache.solr.core.PluginBag.init(PluginBag.java:274)
   [junit4]   2> 	at org.apache.solr.core.RequestHandlers.initHandlersFromConfig(RequestHandlers.java:130)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:946)
   [junit4]   2> 	... 37 more
   [junit4]   2> 
   [junit4]   2> 753683 ERROR (OverseerThreadFactory-3023-thread-5-processing-n:127.0.0.1:58313_) [n:127.0.0.1:58313_    ] o.a.s.c.OverseerCollectionMessageHandler Error from shard: http://127.0.0.1:58313
   [junit4]   2> org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:58313: Error CREATEing SolrCore 'collection2_shard4_replica_n17': Unable to create core [collection2_shard4_replica_n17] Caused by: invalid API spec: apispec/core.SchemaEdit.json
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:627)
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
   [junit4]   2> 	at org.apache.solr.client.solrj.SolrCl

[...truncated too long message...]

int] The import com.codahale.metrics.Histogram is never used
 [ecj-lint] ----------
 [ecj-lint] 13. ERROR in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/metrics/reporters/SolrSlf4jReporter.java (at line 27)
 [ecj-lint] 	import com.codahale.metrics.Meter;
 [ecj-lint] 	       ^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] The import com.codahale.metrics.Meter is never used
 [ecj-lint] ----------
 [ecj-lint] 14. ERROR in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/metrics/reporters/SolrSlf4jReporter.java (at line 29)
 [ecj-lint] 	import com.codahale.metrics.MetricRegistry;
 [ecj-lint] 	       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] The import com.codahale.metrics.MetricRegistry is never used
 [ecj-lint] ----------
 [ecj-lint] 15. ERROR in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/metrics/reporters/SolrSlf4jReporter.java (at line 32)
 [ecj-lint] 	import com.codahale.metrics.Timer;
 [ecj-lint] 	       ^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] The import com.codahale.metrics.Timer is never used
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 16. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/request/SimpleFacets.java (at line 925)
 [ecj-lint] 	fastForRandomSet = new HashDocSet(sset.getDocs(), 0, sset.size());
 [ecj-lint] 	^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: 'fastForRandomSet' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 17. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/response/SmileResponseWriter.java (at line 33)
 [ecj-lint] 	new SmileWriter(out, request, response).writeResponse();
 [ecj-lint] 	^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: '<unassigned Closeable value>' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 18. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/schema/OpenExchangeRatesOrgProvider.java (at line 146)
 [ecj-lint] 	ratesJsonStream = resourceLoader.openResource(ratesFileLocation);
 [ecj-lint] 	^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: 'ratesJsonStream' is not closed at this location
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 19. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java (at line 1315)
 [ecj-lint] 	UninvertingReader uninvertingReader = new UninvertingReader(new ReaderWrapper(searcher.getSlowAtomicReader(), collapseField), mapping);
 [ecj-lint] 	                  ^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: 'uninvertingReader' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 20. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/search/DocSlice.java (at line 145)
 [ecj-lint] 	HashDocSet h = new HashDocSet(docs,offset,len);
 [ecj-lint] 	           ^
 [ecj-lint] Resource leak: 'h' is never closed
 [ecj-lint] ----------
 [ecj-lint] 21. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/search/DocSlice.java (at line 154)
 [ecj-lint] 	HashDocSet h = new HashDocSet(docs,offset,len);
 [ecj-lint] 	           ^
 [ecj-lint] Resource leak: 'h' is never closed
 [ecj-lint] ----------
 [ecj-lint] 22. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/search/DocSlice.java (at line 163)
 [ecj-lint] 	HashDocSet h = new HashDocSet(docs,offset,len);
 [ecj-lint] 	           ^
 [ecj-lint] Resource leak: 'h' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 23. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/search/function/OrdFieldSource.java (at line 79)
 [ecj-lint] 	SolrIndexSearcher is = (SolrIndexSearcher) o;
 [ecj-lint] 	                  ^^
 [ecj-lint] Resource leak: 'is' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 24. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/search/function/ReverseOrdFieldSource.java (at line 79)
 [ecj-lint] 	SolrIndexSearcher is = (SolrIndexSearcher) o;
 [ecj-lint] 	                  ^^
 [ecj-lint] Resource leak: 'is' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 25. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java (at line 291)
 [ecj-lint] 	throw new SolrException(ErrorCode.SERVER_ERROR, "Error occurred while loading solr.xml from zookeeper", e);
 [ecj-lint] 	^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: 'loader' is not closed at this location
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 26. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/update/CdcrTransactionLog.java (at line 98)
 [ecj-lint] 	ChannelFastInputStream is = new ChannelFastInputStream(channel, pos);
 [ecj-lint] 	                       ^^
 [ecj-lint] Resource leak: 'is' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 27. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/update/HdfsTransactionLog.java (at line 191)
 [ecj-lint] 	LogCodec codec = new LogCodec(resolver);
 [ecj-lint] 	         ^^^^^
 [ecj-lint] Resource leak: 'codec' is never closed
 [ecj-lint] ----------
 [ecj-lint] 28. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/update/HdfsTransactionLog.java (at line 261)
 [ecj-lint] 	LogCodec codec = new LogCodec(resolver);
 [ecj-lint] 	         ^^^^^
 [ecj-lint] Resource leak: 'codec' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 29. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/update/TransactionLog.java (at line 228)
 [ecj-lint] 	ChannelFastInputStream is = new ChannelFastInputStream(channel, pos);
 [ecj-lint] 	                       ^^
 [ecj-lint] Resource leak: 'is' is never closed
 [ecj-lint] ----------
 [ecj-lint] 30. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/update/TransactionLog.java (at line 228)
 [ecj-lint] 	ChannelFastInputStream is = new ChannelFastInputStream(channel, pos);
 [ecj-lint] 	                       ^^
 [ecj-lint] Resource leak: 'is' is never closed
 [ecj-lint] ----------
 [ecj-lint] 31. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/update/TransactionLog.java (at line 260)
 [ecj-lint] 	LogCodec codec = new LogCodec(resolver);
 [ecj-lint] 	         ^^^^^
 [ecj-lint] Resource leak: 'codec' is never closed
 [ecj-lint] ----------
 [ecj-lint] 32. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/update/TransactionLog.java (at line 275)
 [ecj-lint] 	LogCodec codec = new LogCodec(resolver);
 [ecj-lint] 	         ^^^^^
 [ecj-lint] Resource leak: 'codec' is never closed
 [ecj-lint] ----------
 [ecj-lint] 33. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/update/TransactionLog.java (at line 529)
 [ecj-lint] 	LogCodec codec = new LogCodec(resolver);
 [ecj-lint] 	         ^^^^^
 [ecj-lint] Resource leak: 'codec' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 34. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/update/processor/UpdateRequestProcessor.java (at line 85)
 [ecj-lint] 	UpdateRequestProcessor p = this;
 [ecj-lint] 	                       ^
 [ecj-lint] Resource leak: 'p' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 35. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/util/SimplePostTool.java (at line 655)
 [ecj-lint] 	BAOS bos =  new BAOS();
 [ecj-lint] 	     ^^^
 [ecj-lint] Resource leak: 'bos' is never closed
 [ecj-lint] ----------
 [ecj-lint] 36. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/util/SimplePostTool.java (at line 655)
 [ecj-lint] 	BAOS bos =  new BAOS();
 [ecj-lint] 	     ^^^
 [ecj-lint] Resource leak: 'bos' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 37. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/util/SolrLogLayout.java (at line 149)
 [ecj-lint] 	SolrQueryRequest req = requestInfo == null ? null : requestInfo.getReq();
 [ecj-lint] 	                 ^^^
 [ecj-lint] Resource leak: 'req' is never closed
 [ecj-lint] ----------
 [ecj-lint] 37 problems (7 errors, 30 warnings)

BUILD FAILED
/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/build.xml:810: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/build.xml:101: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build.xml:689: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/lucene/common-build.xml:2030: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/lucene/common-build.xml:2069: Compile failed; see the compiler error output for details.

Total time: 65 minutes 25 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any

[JENKINS] Lucene-Solr-master-MacOSX (64bit/jdk1.8.0) - Build # 4156 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-MacOSX/4156/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseConcMarkSweepGC

2 tests failed.
FAILED:  org.apache.solr.metrics.reporters.SolrSlf4jReporterTest.testReporter

Error Message:
Test requires that log-level is at-least INFO, but INFO is disabled

Stack Trace:
java.lang.AssertionError: Test requires that log-level is at-least INFO, but INFO is disabled
	at __randomizedtesting.SeedInfo.seed([F730D3B8D3D8EB09:A8D4FE8FB8D4784C]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.metrics.reporters.SolrSlf4jReporterTest.ensureLoggingConfiguredAppropriately(SolrSlf4jReporterTest.java:99)
	at org.apache.solr.metrics.reporters.SolrSlf4jReporterTest.testReporter(SolrSlf4jReporterTest.java:49)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.update.processor.UpdateRequestProcessorFactoryTest.testUpdateDistribChainSkipping

Error Message:
Tests must be run with INFO level logging otherwise LogUpdateProcessor isn't used and can't be tested.

Stack Trace:
java.lang.AssertionError: Tests must be run with INFO level logging otherwise LogUpdateProcessor isn't used and can't be tested.
	at __randomizedtesting.SeedInfo.seed([F730D3B8D3D8EB09:86D42D6EA533CCC5]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.update.processor.UpdateRequestProcessorFactoryTest.testUpdateDistribChainSkipping(UpdateRequestProcessorFactoryTest.java:101)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 12461 lines...]
   [junit4] Suite: org.apache.solr.update.processor.UpdateRequestProcessorFactoryTest
   [junit4]   2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.update.processor.UpdateRequestProcessorFactoryTest_F730D3B8D3D8EB09-001/init-core-data-001
   [junit4]   2> 1868680 WARN  (SUITE-UpdateRequestProcessorFactoryTest-seed#[F730D3B8D3D8EB09]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=3 numCloses=3
   [junit4]   2> 1869068 WARN  (coreLoadExecutor-8235-thread-1) [    x:collection1] o.a.s.c.RequestHandlers no default request handler is registered (either '/select' or 'standard')
   [junit4]   2> 1869074 WARN  (coreLoadExecutor-8235-thread-1) [    x:collection1] o.a.s.r.ManagedResourceStorage Cannot write to config directory /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf; switching to use InMemory storage instead.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=UpdateRequestProcessorFactoryTest -Dtests.method=testUpdateDistribChainSkipping -Dtests.seed=F730D3B8D3D8EB09 -Dtests.slow=true -Dtests.locale=sk -Dtests.timezone=Africa/Johannesburg -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] FAILURE 0.00s J0 | UpdateRequestProcessorFactoryTest.testUpdateDistribChainSkipping <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: Tests must be run with INFO level logging otherwise LogUpdateProcessor isn't used and can't be tested.
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([F730D3B8D3D8EB09:86D42D6EA533CCC5]:0)
   [junit4]    > 	at org.apache.solr.update.processor.UpdateRequestProcessorFactoryTest.testUpdateDistribChainSkipping(UpdateRequestProcessorFactoryTest.java:101)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: leaving temporary files on disk at: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.update.processor.UpdateRequestProcessorFactoryTest_F730D3B8D3D8EB09-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=1034, maxMBSortInHeap=6.119647046488128, sim=RandomSimilarity(queryNorm=true): {}, locale=sk, timezone=Africa/Johannesburg
   [junit4]   2> NOTE: Mac OS X 10.11.6 x86_64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=3,threads=1,free=336548640,total=502566912
   [junit4]   2> NOTE: All tests run in this JVM: [DistanceUnitsTest, JSONWriterTest, TestManagedSchema, TestRuleBasedAuthorizationPlugin, HdfsWriteToMultipleCollectionsTest, TestRandomDVFaceting, SolrTestCaseJ4Test, TestNRTOpen, TestIntervalFaceting, ZkStateWriterTest, TestPushWriter, TestReqParamsAPI, DateRangeFieldTest, SimpleCollectionCreateDeleteTest, TestLMDirichletSimilarityFactory, AnalysisErrorHandlingTest, TolerantUpdateProcessorTest, TestPayloadScoreQParserPlugin, FastVectorHighlighterTest, TestSolrFieldCacheBean, TestDefaultStatsCache, TestHttpShardHandlerFactory, HttpPartitionTest, TestSchemaVersionResource, TestQuerySenderListener, TestReloadDeadlock, HighlighterMaxOffsetTest, TestSolrCloudWithHadoopAuthPlugin, EnumFieldTest, IgnoreCommitOptimizeUpdateProcessorFactoryTest, FieldAnalysisRequestHandlerTest, HLLUtilTest, TestRequestForwarding, TestGraphMLResponseWriter, DirectUpdateHandlerOptimizeTest, DefaultValueUpdateProcessorTest, TestLegacyField, TestFieldTypeResource, DistributedFacetExistsSmallTest, HighlighterConfigTest, PreAnalyzedUpdateProcessorTest, DeleteInactiveReplicaTest, TestScoreJoinQPNoScore, BlockJoinFacetDistribTest, TestConfigsApi, DateMathParserTest, DistributedQueryComponentCustomSortTest, TestLRUStatsCache, SolrMetricManagerTest, TestCloudNestedDocsSort, SyncSliceTest, CollectionTooManyReplicasTest, TestExportWriter, WrapperMergePolicyFactoryTest, TestComplexPhraseQParserPlugin, ShardRoutingTest, TestJsonRequest, AssignBackwardCompatibilityTest, TestSolr4Spatial2, RequestLoggingTest, PeerSyncTest, TestXIncludeConfig, SolrMetricsIntegrationTest, BitVectorTest, TestFieldCacheWithThreads, FullHLLTest, TestFaceting, TestSimpleQParserPlugin, TestInitQParser, TestDocTermOrds, SolrJmxReporterTest, PeerSyncReplicationTest, CoreAdminRequestStatusTest, HdfsTlogReplayBufferedWhileIndexingTest, TestFieldResource, TestQuerySenderNoQuery, TestBinaryField, InfoHandlerTest, NodeMutatorTest, SoftAutoCommitTest, TestDFRSimilarityFactory, TestFunctionQuery, TestJettySolrRunner, TestFoldingMultitermQuery, TestSort, TestSolrConfigHandlerCloud, TestHashPartitioner, TestJsonFacetRefinement, ShufflingReplicaListTransformerTest, TestSchemaManager, XsltUpdateRequestHandlerTest, TestRebalanceLeaders, TestOmitPositions, UUIDFieldTest, HdfsRecoveryZkTest, TestNumericTerms32, TestSubQueryTransformerCrossCore, IndexBasedSpellCheckerTest, TestCloudRecovery, TestSurroundQueryParser, BlockJoinFacetSimpleTest, TestMinMaxOnMultiValuedField, BlobRepositoryCloudTest, TestRawTransformer, TestStressReorder, TestUtils, TestOnReconnectListenerSupport, MoveReplicaHDFSTest, TestCorePropertiesReload, TestXmlQParser, TestInPlaceUpdatesDistrib, DistributedVersionInfoTest, TestMultiValuedNumericRangeQuery, HdfsDirectoryTest, TestStressUserVersions, CursorPagingTest, MultiTermTest, InfixSuggestersTest, URLClassifyProcessorTest, ChaosMonkeySafeLeaderWithPullReplicasTest, TestExclusionRuleCollectionAccess, TestUpdate, TestSolrCLIRunExample, RemoteQueryErrorTest, ChangedSchemaMergeTest, ConfigSetsAPITest, TestApiFramework, TestLockTree, TestDistributedSearch, TestSolrCloudWithDelegationTokens, HdfsChaosMonkeySafeLeaderTest, TestSolrCloudSnapshots, TestReversedWildcardFilterFactory, TestUseDocValuesAsStored, TestLFUCache, TestDFISimilarityFactory, AnalyticsMergeStrategyTest, AlternateDirectoryTest, TestHdfsBackupRestoreCore, CachingDirectoryFactoryTest, UpdateLogTest, TestSystemCollAutoCreate, TestGroupingSearch, MergeStrategyTest, NotRequiredUniqueKeyTest, RequestHandlersTest, BinaryUpdateRequestHandlerTest, TestSolrCoreProperties, TestLeaderElectionWithEmptyReplica, CreateCollectionCleanupTest, ResponseLogComponentTest, TestPolicyCloud, TestCloudManagedSchema, CollectionsAPISolrJTest, CleanupOldIndexTest, TestRandomCollapseQParserPlugin, PreAnalyzedFieldManagedSchemaCloudTest, TestStressInPlaceUpdates, UniqFieldsUpdateProcessorFactoryTest, DistributedFacetPivotSmallTest, AnalysisAfterCoreReloadTest, TestExactSharedStatsCache, ClassificationUpdateProcessorIntegrationTest, TestCloudPivotFacet, HighlighterTest, SubstringBytesRefFilterTest, TestFieldCacheSortRandom, DirectUpdateHandlerTest, TestSolrCloudWithSecureImpersonation, TestManagedSchemaThreadSafety, SolrGraphiteReporterTest, TestShardHandlerFactory, TestConfigOverlay, ActionThrottleTest, TestConfigSetProperties, DistanceFunctionTest, NumericFieldsTest, RecoveryZkTest, ReturnFieldsTest, BasicDistributedZkTest, BasicDistributedZk2Test, UnloadDistributedZkTest, OverseerTest, LeaderElectionIntegrationTest, BasicZkTest, TestReplicationHandler, ZkCLITest, DistributedSpellCheckComponentTest, ZkControllerTest, TestStressVersions, HardAutoCommitTest, TestRangeQuery, TestSolr4Spatial, StatsComponentTest, TestFiltering, TestLazyCores, SolrIndexSplitterTest, SimplePostToolTest, TestCoreDiscovery, SuggesterFSTTest, DocValuesTest, SpatialFilterTest, NoCacheHeaderTest, TestPseudoReturnFields, StatelessScriptUpdateProcessorFactoryTest, SolrInfoBeanTest, TestWriterPerf, FileBasedSpellCheckerTest, DocumentBuilderTest, TestIndexingPerformance, RequiredFieldsTest, IndexSchemaRuntimeFieldTest, RegexBoostProcessorTest, IndexSchemaTest, UpdateRequestProcessorFactoryTest]
   [junit4] Completed [442/732 (1!)] on J0 in 0.43s, 3 tests, 1 failure <<< FAILURES!

[...truncated 686 lines...]
   [junit4] Suite: org.apache.solr.metrics.reporters.SolrSlf4jReporterTest
   [junit4]   2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.metrics.reporters.SolrSlf4jReporterTest_F730D3B8D3D8EB09-001/init-core-data-001
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=SolrSlf4jReporterTest -Dtests.method=testReporter -Dtests.seed=F730D3B8D3D8EB09 -Dtests.slow=true -Dtests.locale=en-GB -Dtests.timezone=Asia/Dubai -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] FAILURE 0.00s J0 | SolrSlf4jReporterTest.testReporter <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: Test requires that log-level is at-least INFO, but INFO is disabled
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([F730D3B8D3D8EB09:A8D4FE8FB8D4784C]:0)
   [junit4]    > 	at org.apache.solr.metrics.reporters.SolrSlf4jReporterTest.ensureLoggingConfiguredAppropriately(SolrSlf4jReporterTest.java:99)
   [junit4]    > 	at org.apache.solr.metrics.reporters.SolrSlf4jReporterTest.testReporter(SolrSlf4jReporterTest.java:49)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: leaving temporary files on disk at: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.metrics.reporters.SolrSlf4jReporterTest_F730D3B8D3D8EB09-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=957, maxMBSortInHeap=6.804603860734156, sim=RandomSimilarity(queryNorm=true): {}, locale=en-GB, timezone=Asia/Dubai
   [junit4]   2> NOTE: Mac OS X 10.11.6 x86_64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=3,threads=1,free=290927016,total=518979584
   [junit4]   2> NOTE: All tests run in this JVM: [DistanceUnitsTest, JSONWriterTest, TestManagedSchema, TestRuleBasedAuthorizationPlugin, HdfsWriteToMultipleCollectionsTest, TestRandomDVFaceting, SolrTestCaseJ4Test, TestNRTOpen, TestIntervalFaceting, ZkStateWriterTest, TestPushWriter, TestReqParamsAPI, DateRangeFieldTest, SimpleCollectionCreateDeleteTest, TestLMDirichletSimilarityFactory, AnalysisErrorHandlingTest, TolerantUpdateProcessorTest, TestPayloadScoreQParserPlugin, FastVectorHighlighterTest, TestSolrFieldCacheBean, TestDefaultStatsCache, TestHttpShardHandlerFactory, HttpPartitionTest, TestSchemaVersionResource, TestQuerySenderListener, TestReloadDeadlock, HighlighterMaxOffsetTest, TestSolrCloudWithHadoopAuthPlugin, EnumFieldTest, IgnoreCommitOptimizeUpdateProcessorFactoryTest, FieldAnalysisRequestHandlerTest, HLLUtilTest, TestRequestForwarding, TestGraphMLResponseWriter, DirectUpdateHandlerOptimizeTest, DefaultValueUpdateProcessorTest, TestLegacyField, TestFieldTypeResource, DistributedFacetExistsSmallTest, HighlighterConfigTest, PreAnalyzedUpdateProcessorTest, DeleteInactiveReplicaTest, TestScoreJoinQPNoScore, BlockJoinFacetDistribTest, TestConfigsApi, DateMathParserTest, DistributedQueryComponentCustomSortTest, TestLRUStatsCache, SolrMetricManagerTest, TestCloudNestedDocsSort, SyncSliceTest, CollectionTooManyReplicasTest, TestExportWriter, WrapperMergePolicyFactoryTest, TestComplexPhraseQParserPlugin, ShardRoutingTest, TestJsonRequest, AssignBackwardCompatibilityTest, TestSolr4Spatial2, RequestLoggingTest, PeerSyncTest, TestXIncludeConfig, SolrMetricsIntegrationTest, BitVectorTest, TestFieldCacheWithThreads, FullHLLTest, TestFaceting, TestSimpleQParserPlugin, TestInitQParser, TestDocTermOrds, SolrJmxReporterTest, PeerSyncReplicationTest, CoreAdminRequestStatusTest, HdfsTlogReplayBufferedWhileIndexingTest, TestFieldResource, TestQuerySenderNoQuery, TestBinaryField, InfoHandlerTest, NodeMutatorTest, SoftAutoCommitTest, TestDFRSimilarityFactory, TestFunctionQuery, TestJettySolrRunner, TestFoldingMultitermQuery, TestSort, TestSolrConfigHandlerCloud, TestHashPartitioner, TestJsonFacetRefinement, ShufflingReplicaListTransformerTest, TestSchemaManager, XsltUpdateRequestHandlerTest, TestRebalanceLeaders, TestOmitPositions, UUIDFieldTest, HdfsRecoveryZkTest, TestNumericTerms32, TestSubQueryTransformerCrossCore, IndexBasedSpellCheckerTest, TestCloudRecovery, TestSurroundQueryParser, BlockJoinFacetSimpleTest, TestMinMaxOnMultiValuedField, BlobRepositoryCloudTest, TestRawTransformer, TestStressReorder, TestUtils, TestOnReconnectListenerSupport, MoveReplicaHDFSTest, TestCorePropertiesReload, TestXmlQParser, TestInPlaceUpdatesDistrib, DistributedVersionInfoTest, TestMultiValuedNumericRangeQuery, HdfsDirectoryTest, TestStressUserVersions, CursorPagingTest, MultiTermTest, InfixSuggestersTest, URLClassifyProcessorTest, ChaosMonkeySafeLeaderWithPullReplicasTest, TestExclusionRuleCollectionAccess, TestUpdate, TestSolrCLIRunExample, RemoteQueryErrorTest, ChangedSchemaMergeTest, ConfigSetsAPITest, TestApiFramework, TestLockTree, TestDistributedSearch, TestSolrCloudWithDelegationTokens, HdfsChaosMonkeySafeLeaderTest, TestSolrCloudSnapshots, TestReversedWildcardFilterFactory, TestUseDocValuesAsStored, TestLFUCache, TestDFISimilarityFactory, AnalyticsMergeStrategyTest, AlternateDirectoryTest, TestHdfsBackupRestoreCore, CachingDirectoryFactoryTest, UpdateLogTest, TestSystemCollAutoCreate, TestGroupingSearch, MergeStrategyTest, NotRequiredUniqueKeyTest, RequestHandlersTest, BinaryUpdateRequestHandlerTest, TestSolrCoreProperties, TestLeaderElectionWithEmptyReplica, CreateCollectionCleanupTest, ResponseLogComponentTest, TestPolicyCloud, TestCloudManagedSchema, CollectionsAPISolrJTest, CleanupOldIndexTest, TestRandomCollapseQParserPlugin, PreAnalyzedFieldManagedSchemaCloudTest, TestStressInPlaceUpdates, UniqFieldsUpdateProcessorFactoryTest, DistributedFacetPivotSmallTest, AnalysisAfterCoreReloadTest, TestExactSharedStatsCache, ClassificationUpdateProcessorIntegrationTest, TestCloudPivotFacet, HighlighterTest, SubstringBytesRefFilterTest, TestFieldCacheSortRandom, DirectUpdateHandlerTest, TestSolrCloudWithSecureImpersonation, TestManagedSchemaThreadSafety, SolrGraphiteReporterTest, TestShardHandlerFactory, TestConfigOverlay, ActionThrottleTest, TestConfigSetProperties, DistanceFunctionTest, NumericFieldsTest, RecoveryZkTest, ReturnFieldsTest, BasicDistributedZkTest, BasicDistributedZk2Test, UnloadDistributedZkTest, OverseerTest, LeaderElectionIntegrationTest, BasicZkTest, TestReplicationHandler, ZkCLITest, DistributedSpellCheckComponentTest, ZkControllerTest, TestStressVersions, HardAutoCommitTest, TestRangeQuery, TestSolr4Spatial, StatsComponentTest, TestFiltering, TestLazyCores, SolrIndexSplitterTest, SimplePostToolTest, TestCoreDiscovery, SuggesterFSTTest, DocValuesTest, SpatialFilterTest, NoCacheHeaderTest, TestPseudoReturnFields, StatelessScriptUpdateProcessorFactoryTest, SolrInfoBeanTest, TestWriterPerf, FileBasedSpellCheckerTest, DocumentBuilderTest, TestIndexingPerformance, RequiredFieldsTest, IndexSchemaRuntimeFieldTest, RegexBoostProcessorTest, IndexSchemaTest, UpdateRequestProcessorFactoryTest, QueryParsingTest, TestPHPSerializedResponseWriter, CSVRequestHandlerTest, TestComponentsName, SOLR749Test, TestSolrIndexConfig, CopyFieldTest, SolrIndexConfigTest, BadComponentTest, TestStressRecovery, TestPhraseSuggestions, TestCodecSupport, TestIBSimilarityFactory, TestFastWriter, ResourceLoaderTest, ScriptEngineTest, TestFastLRUCache, TestSystemIdResolver, PrimUtilsTest, TestSuggestSpellingConverter, DateFieldTest, SpellingQueryConverterTest, RAMDirectoryFactoryTest, ClusterStateTest, TestLRUCache, ZkNodePropsTest, SliceStateTest, CircularListTest, DistributedIntervalFacetingTest, TestCrossCoreJoin, TestCursorMarkWithoutUniqueKey, TestDistributedMissingSort, TestHighlightDedupGrouping, TestTolerantSearch, TestEmbeddedSolrServerAdminHandler, TestEmbeddedSolrServerConstructors, AssignTest, CdcrReplicationDistributedZkTest, CdcrReplicationHandlerTest, CdcrRequestHandlerTest, CdcrVersionReplicationTest, CloudExitableDirectoryReaderTest, CollectionReloadTest, DeleteLastCustomShardedReplicaTest, DeleteReplicaTest, DistribCursorPagingTest, DistribJoinFromCollectionTest, DocValuesNotIndexedTest, ForceLeaderTest, HealthCheckHandlerTest, LeaderElectionContextKeyTest, LeaderInitiatedRecoveryOnCommitTest, MissingSegmentRecoveryTest, MoveReplicaTest, OutOfBoxZkACLAndCredentialsProvidersTest, OverriddenZkACLAndCredentialsProvidersTest, OverseerCollectionConfigSetProcessorTest, OverseerModifyCollectionTest, OverseerRolesTest, OverseerStatusTest, ReplicationFactorTest, RestartWhileUpdatingTest, RollingRestartTest, SolrCloudExampleTest, TestCloudSearcherWarming, TestDeleteCollectionOnDownNodes, TestDistribDocBasedVersion, TestLeaderInitiatedRecoveryThread, TestPullReplicaErrorHandling, TestRandomFlRTGCloud, TestRequestStatusCollectionAPI, TestShortCircuitedRequests, TestStressLiveNodes, TestTlogReplica, HdfsSyncSliceTest, TestClusterStateMutator, RuleEngineTest, CoreSorterTest, DirectoryFactoryTest, ExitableDirectoryReaderTest, QueryResultKeyTest, TestBackupRepositoryFactory, TestConfigSetImmutable, TestInfoStreamLogging, TestInitParams, TestReloadAndDeleteDocs, TestSimpleTextCodec, TestSolrConfigHandler, TestBlobHandler, TestReplicationHandlerBackup, TestSolrConfigHandlerConcurrent, V2StandaloneTest, CoreAdminCreateDiscoverTest, MetricsHandlerTest, PropertiesRequestHandlerTest, SecurityConfHandlerTest, SegmentsInfoRequestHandlerTest, StatsReloadRaceTest, TestCoreAdminApis, DistributedDebugComponentTest, DistributedFacetPivotLargeTest, DistributedFacetPivotWhiteBoxTest, ReplicaListTransformerTest, SuggestComponentContextFilterQueryTest, TestDistributedStatsComponentCardinality, TestNumericRangeQuery64, MetricsConfigTest, SolrSlf4jReporterTest]
   [junit4] Completed [638/732 (2!)] on J0 in 0.03s, 1 test, 1 failure <<< FAILURES!

[...truncated 42603 lines...]
-ecj-javadoc-lint-src:
    [mkdir] Created dir: /var/folders/qg/h2dfw5s161s51l2bn79mrb7r0000gn/T/ecj659167922
 [ecj-lint] Compiling 1124 source files to /var/folders/qg/h2dfw5s161s51l2bn79mrb7r0000gn/T/ecj659167922
 [ecj-lint] invalid Class-Path header in manifest of jar file: /Users/jenkins/.ivy2/cache/org.restlet.jee/org.restlet/jars/org.restlet-2.3.0.jar
 [ecj-lint] invalid Class-Path header in manifest of jar file: /Users/jenkins/.ivy2/cache/org.restlet.jee/org.restlet.ext.servlet/jars/org.restlet.ext.servlet-2.3.0.jar
 [ecj-lint] ----------
 [ecj-lint] 1. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/core/CoreContainer.java (at line 1002)
 [ecj-lint] 	core = new SolrCore(this, dcore, coreConfig);
 [ecj-lint] 	^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: 'core' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 2. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java (at line 234)
 [ecj-lint] 	dir = new BlockDirectory(path, hdfsDir, cache, null, blockCacheReadEnabled, false, cacheMerges, cacheReadOnce);
 [ecj-lint] 	^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: 'dir' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 3. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java (at line 121)
 [ecj-lint] 	reader = cfiltfac.create(reader);
 [ecj-lint] 	^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: 'reader' is not closed at this location
 [ecj-lint] ----------
 [ecj-lint] 4. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java (at line 145)
 [ecj-lint] 	return namedList;
 [ecj-lint] 	^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: 'listBasedTokenStream' is not closed at this location
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 5. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java (at line 1282)
 [ecj-lint] 	DirectoryReader reader = s==null ? null : s.get().getIndexReader();
 [ecj-lint] 	                ^^^^^^
 [ecj-lint] Resource leak: 'reader' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 6. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/handler/sql/SolrTable.java (at line 517)
 [ecj-lint] 	ParallelStream parallelStream = new ParallelStream(zk, collection, tupleStream, numWorkers, comp);
 [ecj-lint] 	               ^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: 'parallelStream' is never closed
 [ecj-lint] ----------
 [ecj-lint] 7. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/handler/sql/SolrTable.java (at line 743)
 [ecj-lint] 	ParallelStream parallelStream = new ParallelStream(zkHost, collection, tupleStream, numWorkers, comp);
 [ecj-lint] 	               ^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: 'parallelStream' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 8. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java (at line 578)
 [ecj-lint] 	tvWindowStream = new OffsetWindowTokenFilter(tvStream);
 [ecj-lint] 	^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: 'tvWindowStream' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 9. ERROR in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/metrics/reporters/SolrSlf4jReporter.java (at line 21)
 [ecj-lint] 	import java.util.SortedMap;
 [ecj-lint] 	       ^^^^^^^^^^^^^^^^^^^
 [ecj-lint] The import java.util.SortedMap is never used
 [ecj-lint] ----------
 [ecj-lint] 10. ERROR in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/metrics/reporters/SolrSlf4jReporter.java (at line 24)
 [ecj-lint] 	import com.codahale.metrics.Counter;
 [ecj-lint] 	       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] The import com.codahale.metrics.Counter is never used
 [ecj-lint] ----------
 [ecj-lint] 11. ERROR in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/metrics/reporters/SolrSlf4jReporter.java (at line 25)
 [ecj-lint] 	import com.codahale.metrics.Gauge;
 [ecj-lint] 	       ^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] The import com.codahale.metrics.Gauge is never used
 [ecj-lint] ----------
 [ecj-lint] 12. ERROR in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/metrics/reporters/SolrSlf4jReporter.java (at line 26)
 [ecj-lint] 	import com.codahale.metrics.Histogram;
 [ecj-lint] 	       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] The import com.codahale.metrics.Histogram is never used
 [ecj-lint] ----------
 [ecj-lint] 13. ERROR in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/metrics/reporters/SolrSlf4jReporter.java (at line 27)
 [ecj-lint] 	import com.codahale.metrics.Meter;
 [ecj-lint] 	       ^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] The import com.codahale.metrics.Meter is never used
 [ecj-lint] ----------
 [ecj-lint] 14. ERROR in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/metrics/reporters/SolrSlf4jReporter.java (at line 29)
 [ecj-lint] 	import com.codahale.metrics.MetricRegistry;
 [ecj-lint] 	       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] The import com.codahale.metrics.MetricRegistry is never used
 [ecj-lint] ----------
 [ecj-lint] 15. ERROR in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/metrics/reporters/SolrSlf4jReporter.java (at line 32)
 [ecj-lint] 	import com.codahale.metrics.Timer;
 [ecj-lint] 	       ^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] The import com.codahale.metrics.Timer is never used
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 16. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/request/SimpleFacets.java (at line 925)
 [ecj-lint] 	fastForRandomSet = new HashDocSet(sset.getDocs(), 0, sset.size());
 [ecj-lint] 	^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: 'fastForRandomSet' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 17. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/response/SmileResponseWriter.java (at line 33)
 [ecj-lint] 	new SmileWriter(out, request, response).writeResponse();
 [ecj-lint] 	^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: '<unassigned Closeable value>' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 18. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/schema/OpenExchangeRatesOrgProvider.java (at line 146)
 [ecj-lint] 	ratesJsonStream = resourceLoader.openResource(ratesFileLocation);
 [ecj-lint] 	^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: 'ratesJsonStream' is not closed at this location
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 19. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java (at line 1315)
 [ecj-lint] 	UninvertingReader uninvertingReader = new UninvertingReader(new ReaderWrapper(searcher.getSlowAtomicReader(), collapseField), mapping);
 [ecj-lint] 	                  ^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: 'uninvertingReader' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 20. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/search/DocSlice.java (at line 145)
 [ecj-lint] 	HashDocSet h = new HashDocSet(docs,offset,len);
 [ecj-lint] 	           ^
 [ecj-lint] Resource leak: 'h' is never closed
 [ecj-lint] ----------
 [ecj-lint] 21. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/search/DocSlice.java (at line 154)
 [ecj-lint] 	HashDocSet h = new HashDocSet(docs,offset,len);
 [ecj-lint] 	           ^
 [ecj-lint] Resource leak: 'h' is never closed
 [ecj-lint] ----------
 [ecj-lint] 22. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/search/DocSlice.java (at line 163)
 [ecj-lint] 	HashDocSet h = new HashDocSet(docs,offset,len);
 [ecj-lint] 	           ^
 [ecj-lint] Resource leak: 'h' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 23. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/search/function/OrdFieldSource.java (at line 79)
 [ecj-lint] 	SolrIndexSearcher is = (SolrIndexSearcher) o;
 [ecj-lint] 	                  ^^
 [ecj-lint] Resource leak: 'is' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 24. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/search/function/ReverseOrdFieldSource.java (at line 79)
 [ecj-lint] 	SolrIndexSearcher is = (SolrIndexSearcher) o;
 [ecj-lint] 	                  ^^
 [ecj-lint] Resource leak: 'is' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 25. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java (at line 291)
 [ecj-lint] 	throw new SolrException(ErrorCode.SERVER_ERROR, "Error occurred while loading solr.xml from zookeeper", e);
 [ecj-lint] 	^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: 'loader' is not closed at this location
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 26. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/update/CdcrTransactionLog.java (at line 98)
 [ecj-lint] 	ChannelFastInputStream is = new ChannelFastInputStream(channel, pos);
 [ecj-lint] 	                       ^^
 [ecj-lint] Resource leak: 'is' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 27. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/update/HdfsTransactionLog.java (at line 191)
 [ecj-lint] 	LogCodec codec = new LogCodec(resolver);
 [ecj-lint] 	         ^^^^^
 [ecj-lint] Resource leak: 'codec' is never closed
 [ecj-lint] ----------
 [ecj-lint] 28. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/update/HdfsTransactionLog.java (at line 261)
 [ecj-lint] 	LogCodec codec = new LogCodec(resolver);
 [ecj-lint] 	         ^^^^^
 [ecj-lint] Resource leak: 'codec' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 29. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/update/TransactionLog.java (at line 228)
 [ecj-lint] 	ChannelFastInputStream is = new ChannelFastInputStream(channel, pos);
 [ecj-lint] 	                       ^^
 [ecj-lint] Resource leak: 'is' is never closed
 [ecj-lint] ----------
 [ecj-lint] 30. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/update/TransactionLog.java (at line 228)
 [ecj-lint] 	ChannelFastInputStream is = new ChannelFastInputStream(channel, pos);
 [ecj-lint] 	                       ^^
 [ecj-lint] Resource leak: 'is' is never closed
 [ecj-lint] ----------
 [ecj-lint] 31. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/update/TransactionLog.java (at line 260)
 [ecj-lint] 	LogCodec codec = new LogCodec(resolver);
 [ecj-lint] 	         ^^^^^
 [ecj-lint] Resource leak: 'codec' is never closed
 [ecj-lint] ----------
 [ecj-lint] 32. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/update/TransactionLog.java (at line 275)
 [ecj-lint] 	LogCodec codec = new LogCodec(resolver);
 [ecj-lint] 	         ^^^^^
 [ecj-lint] Resource leak: 'codec' is never closed
 [ecj-lint] ----------
 [ecj-lint] 33. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/update/TransactionLog.java (at line 529)
 [ecj-lint] 	LogCodec codec = new LogCodec(resolver);
 [ecj-lint] 	         ^^^^^
 [ecj-lint] Resource leak: 'codec' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 34. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/update/processor/UpdateRequestProcessor.java (at line 85)
 [ecj-lint] 	UpdateRequestProcessor p = this;
 [ecj-lint] 	                       ^
 [ecj-lint] Resource leak: 'p' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 35. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/util/SimplePostTool.java (at line 655)
 [ecj-lint] 	BAOS bos =  new BAOS();
 [ecj-lint] 	     ^^^
 [ecj-lint] Resource leak: 'bos' is never closed
 [ecj-lint] ----------
 [ecj-lint] 36. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/util/SimplePostTool.java (at line 655)
 [ecj-lint] 	BAOS bos =  new BAOS();
 [ecj-lint] 	     ^^^
 [ecj-lint] Resource leak: 'bos' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 37. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/util/SolrLogLayout.java (at line 149)
 [ecj-lint] 	SolrQueryRequest req = requestInfo == null ? null : requestInfo.getReq();
 [ecj-lint] 	                 ^^^
 [ecj-lint] Resource leak: 'req' is never closed
 [ecj-lint] ----------
 [ecj-lint] 37 problems (7 errors, 30 warnings)

BUILD FAILED
/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/build.xml:810: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/build.xml:101: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build.xml:689: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/lucene/common-build.xml:2030: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/lucene/common-build.xml:2069: Compile failed; see the compiler error output for details.

Total time: 81 minutes 4 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any

[JENKINS] Lucene-Solr-master-MacOSX (64bit/jdk1.8.0) - Build # 4155 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-MacOSX/4155/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseSerialGC

All tests passed

Build Log:
[...truncated 55789 lines...]
-ecj-javadoc-lint-src:
    [mkdir] Created dir: /var/folders/qg/h2dfw5s161s51l2bn79mrb7r0000gn/T/ecj1888614413
 [ecj-lint] Compiling 1124 source files to /var/folders/qg/h2dfw5s161s51l2bn79mrb7r0000gn/T/ecj1888614413
 [ecj-lint] invalid Class-Path header in manifest of jar file: /Users/jenkins/.ivy2/cache/org.restlet.jee/org.restlet/jars/org.restlet-2.3.0.jar
 [ecj-lint] invalid Class-Path header in manifest of jar file: /Users/jenkins/.ivy2/cache/org.restlet.jee/org.restlet.ext.servlet/jars/org.restlet.ext.servlet-2.3.0.jar
 [ecj-lint] ----------
 [ecj-lint] 1. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/core/CoreContainer.java (at line 1002)
 [ecj-lint] 	core = new SolrCore(this, dcore, coreConfig);
 [ecj-lint] 	^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: 'core' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 2. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/core/HdfsDirectoryFactory.java (at line 234)
 [ecj-lint] 	dir = new BlockDirectory(path, hdfsDir, cache, null, blockCacheReadEnabled, false, cacheMerges, cacheReadOnce);
 [ecj-lint] 	^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: 'dir' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 3. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java (at line 121)
 [ecj-lint] 	reader = cfiltfac.create(reader);
 [ecj-lint] 	^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: 'reader' is not closed at this location
 [ecj-lint] ----------
 [ecj-lint] 4. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java (at line 145)
 [ecj-lint] 	return namedList;
 [ecj-lint] 	^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: 'listBasedTokenStream' is not closed at this location
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 5. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java (at line 1282)
 [ecj-lint] 	DirectoryReader reader = s==null ? null : s.get().getIndexReader();
 [ecj-lint] 	                ^^^^^^
 [ecj-lint] Resource leak: 'reader' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 6. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/handler/sql/SolrTable.java (at line 517)
 [ecj-lint] 	ParallelStream parallelStream = new ParallelStream(zk, collection, tupleStream, numWorkers, comp);
 [ecj-lint] 	               ^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: 'parallelStream' is never closed
 [ecj-lint] ----------
 [ecj-lint] 7. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/handler/sql/SolrTable.java (at line 743)
 [ecj-lint] 	ParallelStream parallelStream = new ParallelStream(zkHost, collection, tupleStream, numWorkers, comp);
 [ecj-lint] 	               ^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: 'parallelStream' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 8. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/highlight/DefaultSolrHighlighter.java (at line 578)
 [ecj-lint] 	tvWindowStream = new OffsetWindowTokenFilter(tvStream);
 [ecj-lint] 	^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: 'tvWindowStream' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 9. ERROR in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/metrics/reporters/SolrSlf4jReporter.java (at line 21)
 [ecj-lint] 	import java.util.SortedMap;
 [ecj-lint] 	       ^^^^^^^^^^^^^^^^^^^
 [ecj-lint] The import java.util.SortedMap is never used
 [ecj-lint] ----------
 [ecj-lint] 10. ERROR in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/metrics/reporters/SolrSlf4jReporter.java (at line 24)
 [ecj-lint] 	import com.codahale.metrics.Counter;
 [ecj-lint] 	       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] The import com.codahale.metrics.Counter is never used
 [ecj-lint] ----------
 [ecj-lint] 11. ERROR in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/metrics/reporters/SolrSlf4jReporter.java (at line 25)
 [ecj-lint] 	import com.codahale.metrics.Gauge;
 [ecj-lint] 	       ^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] The import com.codahale.metrics.Gauge is never used
 [ecj-lint] ----------
 [ecj-lint] 12. ERROR in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/metrics/reporters/SolrSlf4jReporter.java (at line 26)
 [ecj-lint] 	import com.codahale.metrics.Histogram;
 [ecj-lint] 	       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] The import com.codahale.metrics.Histogram is never used
 [ecj-lint] ----------
 [ecj-lint] 13. ERROR in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/metrics/reporters/SolrSlf4jReporter.java (at line 27)
 [ecj-lint] 	import com.codahale.metrics.Meter;
 [ecj-lint] 	       ^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] The import com.codahale.metrics.Meter is never used
 [ecj-lint] ----------
 [ecj-lint] 14. ERROR in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/metrics/reporters/SolrSlf4jReporter.java (at line 29)
 [ecj-lint] 	import com.codahale.metrics.MetricRegistry;
 [ecj-lint] 	       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] The import com.codahale.metrics.MetricRegistry is never used
 [ecj-lint] ----------
 [ecj-lint] 15. ERROR in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/metrics/reporters/SolrSlf4jReporter.java (at line 32)
 [ecj-lint] 	import com.codahale.metrics.Timer;
 [ecj-lint] 	       ^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] The import com.codahale.metrics.Timer is never used
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 16. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/request/SimpleFacets.java (at line 925)
 [ecj-lint] 	fastForRandomSet = new HashDocSet(sset.getDocs(), 0, sset.size());
 [ecj-lint] 	^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: 'fastForRandomSet' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 17. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/response/SmileResponseWriter.java (at line 33)
 [ecj-lint] 	new SmileWriter(out, request, response).writeResponse();
 [ecj-lint] 	^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: '<unassigned Closeable value>' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 18. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/schema/OpenExchangeRatesOrgProvider.java (at line 146)
 [ecj-lint] 	ratesJsonStream = resourceLoader.openResource(ratesFileLocation);
 [ecj-lint] 	^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: 'ratesJsonStream' is not closed at this location
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 19. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java (at line 1315)
 [ecj-lint] 	UninvertingReader uninvertingReader = new UninvertingReader(new ReaderWrapper(searcher.getSlowAtomicReader(), collapseField), mapping);
 [ecj-lint] 	                  ^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: 'uninvertingReader' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 20. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/search/DocSlice.java (at line 145)
 [ecj-lint] 	HashDocSet h = new HashDocSet(docs,offset,len);
 [ecj-lint] 	           ^
 [ecj-lint] Resource leak: 'h' is never closed
 [ecj-lint] ----------
 [ecj-lint] 21. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/search/DocSlice.java (at line 154)
 [ecj-lint] 	HashDocSet h = new HashDocSet(docs,offset,len);
 [ecj-lint] 	           ^
 [ecj-lint] Resource leak: 'h' is never closed
 [ecj-lint] ----------
 [ecj-lint] 22. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/search/DocSlice.java (at line 163)
 [ecj-lint] 	HashDocSet h = new HashDocSet(docs,offset,len);
 [ecj-lint] 	           ^
 [ecj-lint] Resource leak: 'h' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 23. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/search/function/OrdFieldSource.java (at line 79)
 [ecj-lint] 	SolrIndexSearcher is = (SolrIndexSearcher) o;
 [ecj-lint] 	                  ^^
 [ecj-lint] Resource leak: 'is' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 24. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/search/function/ReverseOrdFieldSource.java (at line 79)
 [ecj-lint] 	SolrIndexSearcher is = (SolrIndexSearcher) o;
 [ecj-lint] 	                  ^^
 [ecj-lint] Resource leak: 'is' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 25. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java (at line 291)
 [ecj-lint] 	throw new SolrException(ErrorCode.SERVER_ERROR, "Error occurred while loading solr.xml from zookeeper", e);
 [ecj-lint] 	^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: 'loader' is not closed at this location
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 26. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/update/CdcrTransactionLog.java (at line 98)
 [ecj-lint] 	ChannelFastInputStream is = new ChannelFastInputStream(channel, pos);
 [ecj-lint] 	                       ^^
 [ecj-lint] Resource leak: 'is' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 27. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/update/HdfsTransactionLog.java (at line 191)
 [ecj-lint] 	LogCodec codec = new LogCodec(resolver);
 [ecj-lint] 	         ^^^^^
 [ecj-lint] Resource leak: 'codec' is never closed
 [ecj-lint] ----------
 [ecj-lint] 28. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/update/HdfsTransactionLog.java (at line 261)
 [ecj-lint] 	LogCodec codec = new LogCodec(resolver);
 [ecj-lint] 	         ^^^^^
 [ecj-lint] Resource leak: 'codec' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 29. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/update/TransactionLog.java (at line 228)
 [ecj-lint] 	ChannelFastInputStream is = new ChannelFastInputStream(channel, pos);
 [ecj-lint] 	                       ^^
 [ecj-lint] Resource leak: 'is' is never closed
 [ecj-lint] ----------
 [ecj-lint] 30. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/update/TransactionLog.java (at line 228)
 [ecj-lint] 	ChannelFastInputStream is = new ChannelFastInputStream(channel, pos);
 [ecj-lint] 	                       ^^
 [ecj-lint] Resource leak: 'is' is never closed
 [ecj-lint] ----------
 [ecj-lint] 31. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/update/TransactionLog.java (at line 260)
 [ecj-lint] 	LogCodec codec = new LogCodec(resolver);
 [ecj-lint] 	         ^^^^^
 [ecj-lint] Resource leak: 'codec' is never closed
 [ecj-lint] ----------
 [ecj-lint] 32. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/update/TransactionLog.java (at line 275)
 [ecj-lint] 	LogCodec codec = new LogCodec(resolver);
 [ecj-lint] 	         ^^^^^
 [ecj-lint] Resource leak: 'codec' is never closed
 [ecj-lint] ----------
 [ecj-lint] 33. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/update/TransactionLog.java (at line 529)
 [ecj-lint] 	LogCodec codec = new LogCodec(resolver);
 [ecj-lint] 	         ^^^^^
 [ecj-lint] Resource leak: 'codec' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 34. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/update/processor/UpdateRequestProcessor.java (at line 85)
 [ecj-lint] 	UpdateRequestProcessor p = this;
 [ecj-lint] 	                       ^
 [ecj-lint] Resource leak: 'p' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 35. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/util/SimplePostTool.java (at line 655)
 [ecj-lint] 	BAOS bos =  new BAOS();
 [ecj-lint] 	     ^^^
 [ecj-lint] Resource leak: 'bos' is never closed
 [ecj-lint] ----------
 [ecj-lint] 36. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/util/SimplePostTool.java (at line 655)
 [ecj-lint] 	BAOS bos =  new BAOS();
 [ecj-lint] 	     ^^^
 [ecj-lint] Resource leak: 'bos' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 37. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/util/SolrLogLayout.java (at line 149)
 [ecj-lint] 	SolrQueryRequest req = requestInfo == null ? null : requestInfo.getReq();
 [ecj-lint] 	                 ^^^
 [ecj-lint] Resource leak: 'req' is never closed
 [ecj-lint] ----------
 [ecj-lint] 37 problems (7 errors, 30 warnings)

BUILD FAILED
/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/build.xml:810: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/build.xml:101: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build.xml:689: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/lucene/common-build.xml:2030: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/lucene/common-build.xml:2069: Compile failed; see the compiler error output for details.

Total time: 81 minutes 49 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any

[JENKINS] Lucene-Solr-master-MacOSX (64bit/jdk1.8.0) - Build # 4154 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-MacOSX/4154/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseParallelGC

3 tests failed.
FAILED:  org.apache.solr.cloud.CustomCollectionTest.testCustomCollectionsAPI

Error Message:
Could not find collection : implicitcoll

Stack Trace:
org.apache.solr.common.SolrException: Could not find collection : implicitcoll
	at __randomizedtesting.SeedInfo.seed([688EE4121998BF56:26F6A792402092E]:0)
	at org.apache.solr.common.cloud.ClusterState.getCollection(ClusterState.java:109)
	at org.apache.solr.cloud.SolrCloudTestCase.getCollectionState(SolrCloudTestCase.java:247)
	at org.apache.solr.cloud.CustomCollectionTest.testCustomCollectionsAPI(CustomCollectionTest.java:68)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.CustomCollectionTest.testRouteFieldForImplicitRouter

Error Message:
Collection not found: withShardField

Stack Trace:
org.apache.solr.common.SolrException: Collection not found: withShardField
	at __randomizedtesting.SeedInfo.seed([688EE4121998BF56:3DDE0C80B56170A6]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.getCollectionNames(CloudSolrClient.java:1139)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:822)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
	at org.apache.solr.client.solrj.request.UpdateRequest.commit(UpdateRequest.java:233)
	at org.apache.solr.cloud.CustomCollectionTest.testRouteFieldForImplicitRouter(CustomCollectionTest.java:141)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.update.processor.UpdateRequestProcessorFactoryTest.testUpdateDistribChainSkipping

Error Message:
Tests must be run with INFO level logging otherwise LogUpdateProcessor isn't used and can't be tested.

Stack Trace:
java.lang.AssertionError: Tests must be run with INFO level logging otherwise LogUpdateProcessor isn't used and can't be tested.
	at __randomizedtesting.SeedInfo.seed([688EE4121998BF56:196A1AC46F73989A]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.update.processor.UpdateRequestProcessorFactoryTest.testUpdateDistribChainSkipping(UpdateRequestProcessorFactoryTest.java:101)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 12508 lines...]
   [junit4] Suite: org.apache.solr.update.processor.UpdateRequestProcessorFactoryTest
   [junit4]   2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.update.processor.UpdateRequestProcessorFactoryTest_688EE4121998BF56-001/init-core-data-001
   [junit4]   2> 2265326 WARN  (SUITE-UpdateRequestProcessorFactoryTest-seed#[688EE4121998BF56]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=2 numCloses=2
   [junit4]   2> 2265649 WARN  (coreLoadExecutor-9879-thread-1) [    x:collection1] o.a.s.c.RequestHandlers no default request handler is registered (either '/select' or 'standard')
   [junit4]   2> 2265650 WARN  (coreLoadExecutor-9879-thread-1) [    x:collection1] o.a.s.r.ManagedResourceStorage Cannot write to config directory /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf; switching to use InMemory storage instead.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=UpdateRequestProcessorFactoryTest -Dtests.method=testUpdateDistribChainSkipping -Dtests.seed=688EE4121998BF56 -Dtests.slow=true -Dtests.locale=sq -Dtests.timezone=America/Goose_Bay -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] FAILURE 0.00s J0 | UpdateRequestProcessorFactoryTest.testUpdateDistribChainSkipping <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: Tests must be run with INFO level logging otherwise LogUpdateProcessor isn't used and can't be tested.
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([688EE4121998BF56:196A1AC46F73989A]:0)
   [junit4]    > 	at org.apache.solr.update.processor.UpdateRequestProcessorFactoryTest.testUpdateDistribChainSkipping(UpdateRequestProcessorFactoryTest.java:101)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: leaving temporary files on disk at: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.update.processor.UpdateRequestProcessorFactoryTest_688EE4121998BF56-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=682, maxMBSortInHeap=5.584065913624708, sim=RandomSimilarity(queryNorm=false): {}, locale=sq, timezone=America/Goose_Bay
   [junit4]   2> NOTE: Mac OS X 10.11.6 x86_64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=3,threads=1,free=169664656,total=531103744
   [junit4]   2> NOTE: All tests run in this JVM: [TestNumericTerms32, TestSchemaResource, SolrMetricsIntegrationTest, TestChildDocTransformer, BasicAuthStandaloneTest, ClusterStateUpdateTest, InfixSuggestersTest, TestSolrConfigHandlerCloud, TestCoreContainer, TestTlogReplica, TestDistributedStatsComponentCardinality, BlockDirectoryTest, ChaosMonkeyNothingIsSafeTest, QueryElevationComponentTest, TestSubQueryTransformerCrossCore, DateMathParserTest, TestJsonFacetRefinement, DistribCursorPagingTest, TestReRankQParserPlugin, SolrIndexSplitterTest, SolrIndexMetricsTest, CursorPagingTest, SuggestComponentContextFilterQueryTest, MetricsConfigTest, BinaryUpdateRequestHandlerTest, RequestLoggingTest, SyncSliceTest, TestPayloadScoreQParserPlugin, BlockJoinFacetSimpleTest, TestMinMaxOnMultiValuedField, BigEndianAscendingWordDeserializerTest, OverriddenZkACLAndCredentialsProvidersTest, TestDefaultStatsCache, TestReversedWildcardFilterFactory, TestBinaryResponseWriter, DocValuesTest, TestStressReorder, ShardRoutingTest, FieldAnalysisRequestHandlerTest, TestRuleBasedAuthorizationPlugin, DistributedQueryComponentCustomSortTest, XsltUpdateRequestHandlerTest, PreAnalyzedUpdateProcessorTest, ReplicationFactorTest, HdfsChaosMonkeySafeLeaderTest, UniqFieldsUpdateProcessorFactoryTest, RollingRestartTest, TestDFRSimilarityFactory, TestRandomCollapseQParserPlugin, EnumFieldTest, TolerantUpdateProcessorTest, TestCorePropertiesReload, TestCloudPivotFacet, SolrCloudReportersTest, SortByFunctionTest, AnalysisAfterCoreReloadTest, SolrMetricManagerTest, ZkStateWriterTest, TestLazyCores, TestRebalanceLeaders, TestConfigsApi, TestBinaryField, TestIndexingPerformance, NotRequiredUniqueKeyTest, HdfsWriteToMultipleCollectionsTest, TestHighFrequencyDictionaryFactory, CoreAdminRequestStatusTest, TestCollationFieldDocValues, V2StandaloneTest, TestWriterPerf, TestManagedSchemaThreadSafety, TestPullReplica, SuggesterWFSTTest, TestUninvertingReader, TestQuerySenderNoQuery, MinimalSchemaTest, TestLockTree, TestDistribDocBasedVersion, TestFieldCacheWithThreads, TestDistributedSearch, DistributedSpellCheckComponentTest, CollectionTooManyReplicasTest, HighlighterConfigTest, TestPolicyCloud, TestRandomDVFaceting, DateRangeFieldTest, DefaultValueUpdateProcessorTest, TestFoldingMultitermQuery, DistributedSuggestComponentTest, TestSolrDeletionPolicy2, SolrCoreTest, QueryParsingTest, AssignBackwardCompatibilityTest, AtomicUpdatesTest, ChaosMonkeySafeLeaderWithPullReplicasTest, ZkControllerTest, TestRequestForwarding, TestInitQParser, TestXIncludeConfig, TestCloudRecovery, HighlighterTest, HdfsDirectoryTest, TestCryptoKeys, UUIDFieldTest, TestTrackingShardHandlerFactory, AsyncCallRequestStatusResponseTest, TestInfoStreamLogging, TestSolr4Spatial2, TestConfigOverlay, CdcrVersionReplicationTest, TestIntervalFaceting, HLLUtilTest, CloneFieldUpdateProcessorFactoryTest, TestSystemCollAutoCreate, SpatialRPTFieldTypeTest, SolrInfoBeanTest, TestFuzzyAnalyzedSuggestions, TestManagedSynonymFilterFactory, TestCodecSupport, RegexBoostProcessorTest, TestJmxIntegration, TestLegacyField, TestFieldCacheReopen, QueryResultKeyTest, DistanceUnitsTest, TestGroupingSearch, ForceLeaderTest, HttpSolrCallGetCoreTest, ClassificationUpdateProcessorIntegrationTest, MigrateRouteKeyTest, TestPerFieldSimilarityWithDefaultOverride, TestXmlQParser, TestSweetSpotSimilarityFactory, TestIBSimilarityFactory, TestPhraseSuggestions, LeaderFailoverAfterPartitionTest, CleanupOldIndexTest, TestSQLHandler, TestShardHandlerFactory, HdfsTlogReplayBufferedWhileIndexingTest, TestOmitPositions, ActionThrottleTest, DistributedFacetPivotLongTailTest, BitVectorTest, TestTolerantSearch, SimpleCollectionCreateDeleteTest, TestLeaderInitiatedRecoveryThread, TestCloudManagedSchema, TestAuthenticationFramework, UtilsToolTest, TestSolrCloudWithHadoopAuthPlugin, DeleteShardTest, TestSimpleQParserPlugin, SolrCloudExampleTest, TestMacroExpander, WrapperMergePolicyFactoryTest, RuleEngineTest, PeerSyncReplicationTest, MoveReplicaHDFSTest, SimplePostToolTest, ConfigSetsAPITest, PreAnalyzedFieldManagedSchemaCloudTest, TestQuerySenderListener, MetricUtilsTest, TestCursorMarkWithoutUniqueKey, ChangedSchemaMergeTest, LeaderElectionTest, TestManagedSchema, TestExtendedDismaxParser, SubstringBytesRefFilterTest, UnloadDistributedZkTest, SharedFSAutoReplicaFailoverTest, TestFieldResource, TestAtomicUpdateErrorCases, RequiredFieldsTest, TestObjectReleaseTracker, ZkFailoverTest, TestCollapseQParserPlugin, JSONWriterTest, BasicDistributedZk2Test, CollectionsAPIDistributedZkTest, OpenCloseCoreStressTest, OverseerTest, LeaderElectionIntegrationTest, BasicZkTest, RecoveryZkTest, FullSolrCloudDistribCmdsTest, TestReplicationHandler, TestRandomFaceting, ZkSolrClientTest, TestZkChroot, ZkCLITest, AutoCommitTest, TestFaceting, TestHashPartitioner, SimpleFacetsTest, PeerSyncTest, CoreAdminHandlerTest, SolrRequestParserTest, SuggesterTest, TestStressLucene, TestTrie, SpatialFilterTest, TestCSVLoader, PolyFieldTest, NoCacheHeaderTest, WordBreakSolrSpellCheckerTest, SchemaVersionSpecificBehaviorTest, SolrCoreCheckLockOnStartupTest, TestPseudoReturnFields, TestUpdate, FieldMutatingUpdateProcessorTest, DirectUpdateHandlerOptimizeTest, StatelessScriptUpdateProcessorFactoryTest, TestRemoteStreaming, DocValuesMultiTest, DistanceFunctionTest, DebugComponentTest, CacheHeaderTest, IndexBasedSpellCheckerTest, TestSurroundQueryParser, DisMaxRequestHandlerTest, DirectSolrSpellCheckerTest, TestQueryTypes, PrimitiveFieldTypeTest, FileBasedSpellCheckerTest, XmlUpdateRequestHandlerTest, RequestHandlersTest, TestValueSourceCache, PathHierarchyTokenizerFactoryTest, TermVectorComponentTest, MoreLikeThisHandlerTest, FastVectorHighlighterTest, SolrPluginUtilsTest, IndexSchemaTest, ReturnFieldsTest, UpdateRequestProcessorFactoryTest]
   [junit4] Completed [436/732 (1!)] on J0 in 0.36s, 3 tests, 1 failure <<< FAILURES!

[...truncated 209 lines...]
   [junit4] Suite: org.apache.solr.cloud.CustomCollectionTest
   [junit4]   2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_688EE4121998BF56-001/init-core-data-001
   [junit4]   2> 2446691 ERROR (Thread-4168) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 2446839 ERROR (jetty-launcher-5433-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2446839 ERROR (jetty-launcher-5433-thread-2) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2446839 ERROR (jetty-launcher-5433-thread-4) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2446840 ERROR (jetty-launcher-5433-thread-3) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2446852 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15e2ddbd5ee0003, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 2446857 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15e2ddbd5ee0005, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 2446861 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15e2ddbd5ee0008, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 2451783 WARN  (zkCallback-5455-thread-1-processing-n:127.0.0.1:59472_solr) [n:127.0.0.1:59472_solr    ] o.a.s.c.LeaderElector 
   [junit4]   2> org.apache.solr.common.SolrException: SolrCore not found:routeFieldColl_shard2_replica_n4 in []
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:297)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:56)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:348)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$3.lambda$process$0(SolrZkClient.java:269)
   [junit4]   2> 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 2451787 WARN  (zkCallback-5455-thread-1-processing-n:127.0.0.1:59472_solr) [n:127.0.0.1:59472_solr    ] o.a.s.c.LeaderElector Our node is no longer in line to be leader
   [junit4]   2> 2451792 WARN  (zkCallback-5454-thread-1-processing-n:127.0.0.1:59471_solr) [n:127.0.0.1:59471_solr    ] o.a.s.c.LeaderElector 
   [junit4]   2> org.apache.solr.common.SolrException: SolrCore not found:routeFieldColl_shard1_replica_n1 in []
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:297)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:56)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:348)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$3.lambda$process$0(SolrZkClient.java:269)
   [junit4]   2> 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 2451795 WARN  (zkCallback-5454-thread-1-processing-n:127.0.0.1:59471_solr) [n:127.0.0.1:59471_solr    ] o.a.s.c.LeaderElector Our node is no longer in line to be leader
   [junit4]   2> 2457687 WARN  (OverseerThreadFactory-10622-thread-5-processing-n:127.0.0.1:59473_solr) [n:127.0.0.1:59473_solr    ] o.a.s.c.CreateCollectionCmd Specified number of replicas of 5 on collection implicitcoll is higher than the number of Solr instances currently live or live and part of your createNodeSet(4). It's unusual to run two replica of the same slice on the same Solr-instance.
   [junit4]   2> 2459901 ERROR (qtp171749319-23945) [n:127.0.0.1:59472_solr c:implicitcoll s:c r:core_node30 x:implicitcoll_c_replica_n27] o.a.s.h.RequestHandlerBase org.apache.solr.common.SolrException: Error CREATEing SolrCore 'implicitcoll_c_replica_n27': Unable to create core [implicitcoll_c_replica_n27] Caused by: invalid API spec: apispec/core.RealtimeGet.json
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:971)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)
   [junit4]   2> 	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:745)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:726)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:507)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: Unable to create core [implicitcoll_c_replica_n27]
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1018)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:938)
   [junit4]   2> 	... 37 more
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: invalid API spec: apispec/core.RealtimeGet.json
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:988)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:843)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1002)
   [junit4]   2> 	... 38 more
   [junit4]   2> Caused by: java.lang.RuntimeException: invalid API spec: apispec/core.RealtimeGet.json
   [junit4]   2> 	at org.apache.solr.common.util.ValidatingJsonMap.parse(ValidatingJsonMap.java:318)
   [junit4]   2> 	at org.apache.solr.common.util.Utils.lambda$getSpec$0(Utils.java:394)
   [junit4]   2> 	at org.apache.solr.api.Api.getSpec(Api.java:65)
   [junit4]   2> 	at org.apache.solr.api.ApiBag.register(ApiBag.java:73)
   [junit4]   2> 	at org.apache.solr.core.PluginBag.put(PluginBag.java:217)
   [junit4]   2> 	at org.apache.solr.core.PluginBag.init(PluginBag.java:274)
   [junit4]   2> 	at org.apache.solr.core.RequestHandlers.initHandlersFromConfig(RequestHandlers.java:130)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:946)
   [junit4]   2> 	... 40 more
   [junit4]   2> 
   [junit4]   2> 2459902 ERROR (OverseerThreadFactory-10622-thread-5-processing-n:127.0.0.1:59473_solr) [n:127.0.0.1:59473_solr    ] o.a.s.c.OverseerCollectionMessageHandler Error from shard: https://127.0.0.1:59472/solr
   [junit4]   2> org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:59472/solr: Error CREATEing SolrCore 'implicitcoll_c_replica_n27': Unable to create core [implicitcoll_c_replica_n27] Caused by: invalid API spec: apispec/core.RealtimeGet.json
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:627)
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
   [junit4]   2> 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
   [junit4]   2> 	at org.apache.solr.handler.component.HttpShardHandler.lambda$submit$0(HttpShardHandler.java:172)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 2459906 ERROR (qtp171749319-23946) [n:127.0.0.1:59472_solr c:implicitcoll s:c r:core_node24 x:implicitcoll_c_replica_n20] o.a.s.h.RequestHandlerBase org.apache.solr.common.SolrException: Error CREATEing SolrCore 'implicitcoll_c_replica_n20': Unable to create core [implicitcoll_c_replica_n20] Caused by: invalid API spec: apispec/emptySpec.json
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:971)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)
   [junit4]   2> 	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:745)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:726)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:507)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: Unable to create core [implicitcoll_c_replica_n20]
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1018)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:938)
   [junit4]   2> 	... 37 more
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: invalid API spec: apispec/emptySpec.json
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:988)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:843)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1002)
   [junit4]   2> 	... 38 more
   [junit4]   2> Caused by: java.lang.RuntimeException: invalid API spec: apispec/emptySpec.json
   [junit4]   2> 	at org.apache.solr.common.util.ValidatingJsonMap.parse(ValidatingJsonMap.java:318)
   [junit4]   2> 	at org.apache.solr.common.util.Utils.lambda$getSpec$0(Utils.java:394)
   [junit4]   2> 	at org.apache.solr.api.Api.getSpec(Api.java:65)
   [junit4]   2> 	at org.apache.solr.api.ApiBag.register(ApiBag.java:73)
   [junit4]   2> 	at org.apache.solr.core.PluginBag.put(PluginBag.java:217)
   [junit4]   2> 	at org.apache.solr.core.PluginBag.init(PluginBag.java:274)
   [junit4]   2> 	at org.apache.solr.core.RequestHandlers.initHandlersFromConfig(RequestHandlers.java:130)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:946)
   [junit4]   2> 	... 40 more
   [junit4]   2> 
   [junit4]   2> 2459906 ERROR (OverseerThreadFactory-10622-thread-5-processing-n:127.0.0.1:59473_solr) [n:127.0.0.1:59473_solr    ] o.a.s.c.OverseerCollectionMessageHandler Error from shard: https://127.0.0.1:59472/solr
   [junit4]   2> org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:59472/solr: Error CREATEing SolrCore 'implicitcoll_c_replica_n20': Unable to create core [implicitcoll_c_replica_n20] Caused by: invalid API spec: apispec/emptySpec.json
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:627)
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
   [junit4]   2> 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
   [junit4]   2> 	at org.apache.solr.handler.component.HttpShardHandler.lambda$submit$0(HttpShardHandler.java:172)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 2470031 ERROR (qtp171749319-23944) [n:127.0.0.1:59472_solr    ] o.a.s.s.HttpSolrCall null:org.apache.solr.core.SolrCoreInitializationException: SolrCore 'implicitcoll_c_replica_n20' is not available due to init failure: invalid API spec: apispec/emptySpec.json
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1425)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:267)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:480)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: invalid API spec: apispec/emptySpec.json
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:988)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:843)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1002)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:938)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)
   [junit4]   2> 	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:745)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:726)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:507)
   [junit4]   2> 	... 29 more
   [junit4]   2> Caused by: java.lang.RuntimeException: invalid API spec: apispec/emptySpec.json
   [junit4]   2> 	at org.apache.solr.common.util.ValidatingJsonMap.parse(ValidatingJsonMap.java:318)
   [junit4]   2> 	at org.apache.solr.common.util.Utils.lambda$getSpec$0(Utils.java:394)
   [junit4]   2> 	at org.apache.solr.api.Api.getSpec(Api.java:65)
   [junit4]   2> 	at org.apache.solr.api.ApiBag.register(ApiBag.java:73)
   [junit4]   2> 	at org.apache.solr.core.PluginBag.put(PluginBag.java:217)
   [junit4]   2> 	at org.apache.solr.core.PluginBag.init(PluginBag.java:274)
   [junit4]   2> 	at org.apache.solr.core.RequestHandlers.initHandlersFromConfig(RequestHandlers.java:130)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:946)
   [junit4]   2> 	... 40 more
   [junit4]   2> 
   [junit4]   2> 2470033 ERROR (qtp171749319-23943) [n:127.0.0.1:59472_solr    ] o.a.s.s.HttpSolrCall null:org.apache.solr.core.SolrCoreInitializationException: SolrCore 'implicitcoll_c_replica_n27' is not available due to init failure: invalid API spec: apispec/core.RealtimeGet.json
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1425)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:267)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:480)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: invalid API spec: apispec/core.RealtimeGet.json
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:988)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:843)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1002)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:938)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)
   [junit4]   2> 	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:745)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:726)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:507)
   [junit4]   2> 	... 29 more
   [junit4]   2> Caused by: java.lang.RuntimeException: invalid API spec: apispec/core.RealtimeGet.json
   [junit4]   2> 	at org.apache.solr.common.util.ValidatingJsonMap.parse(ValidatingJsonMap.java:318)
   [junit4]   2> 	at org.apache.solr.common.util.Utils.lambda$getSpec$0(Utils.java:394)
   [junit4]   2> 	at org.apache.solr.api.Api.getSpec(Api.java:65)
   [junit4]   2> 	at org.apache.solr.api.ApiBag.register(ApiBag.java:73)
   [junit4]   2> 	at org.apache.solr.core.PluginBag.put(PluginBag.java:217)
   [junit4]   2> 	at org.apache.solr.core.PluginBag.init(PluginBag.java:274)
   [junit4]   2> 	at org.apache.solr.core.RequestHandlers.initHandlersFromConfig(RequestHandlers.java:130)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:946)
   [junit4]   2> 	... 40 more
   [junit4]   2> 
   [junit4]   2> 2470981 ERROR (OverseerThreadFactory-10622-thread-5-processing-n:127.0.0.1:59473_solr) [n:127.0.0.1:59473_solr    ] o.a.s.c.OverseerCollectionMessageHandler Cleaning up collection [implicitcoll].
   [junit4]   2> 2471002 ERROR (qtp171749319-23942) [n:127.0.0.1:59472_solr    ] o.a.s.c.SolrCore Failed to delete instance dir for unloaded core: implicitcoll_c_replica_n27 dir: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_688EE4121998BF56-001/tempDir-001/node2/./implicitcoll_c_replica_n27
   [junit4]   2> java.io.IOException: Failed to list contents of /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_688EE4121998BF56-001/tempDir-001/node2/./implicitcoll_c_replica_n27
   [junit4]   2> 	at org.apache.commons.io.FileUtils.verifiedListFiles(FileUtils.java:1709)
   [junit4]   2> 	at org.apache.commons.io.FileUtils.cleanDirectory(FileUtils.java:1674)
   [junit4]   2> 	at org.apache.commons.io.FileUtils.deleteDirectory(FileUtils.java:1575)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.deleteUnloadedCore(SolrCore.java:2874)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.unload(CoreContainer.java:1309)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$1(CoreAdminOperation.java:101)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)
   [junit4]   2> 	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:745)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:726)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:507)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 2471002 ERROR (qtp171749319-23946) [n:127.0.0.1:59472_solr    ] o.a.s.c.SolrCore Failed to delete instance dir for unloaded core: implicitcoll_c_replica_n20 dir: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_688EE4121998BF56-001/tempDir-001/node2/./implicitcoll_c_replica_n20
   [junit4]   2> java.io.IOException: Failed to list contents of /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_688EE4121998BF56-001/tempDir-001/node2/./implicitcoll_c_replica_n20
   [junit4]   2> 	at org.apache.commons.io.FileUtils.verifiedListFiles(FileUtils.java:1709)
   [junit4]   2> 	at org.apache.commons.io.FileUtils.cleanDirectory(FileUtils.java:1674)
   [junit4]   2> 	at org.apache.commons.io.FileUtils.deleteDirectory(FileUtils.java:1575)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.deleteUnloadedCore(SolrCore.java:2874)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.unload(CoreContainer.java:1309)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$1(CoreAdminOperation.java:101)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)
   [junit4]   2> 	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:745)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:726)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:507)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 2471525 WARN  (zkCallback-5455-thread-1-processing-n:127.0.0.1:59472_solr) [n:127.0.0.1:59472_solr    ] o.a.s.c.LeaderElector Our node is no longer in line to be leader
   [junit4]   2> 2471530 WARN  (zkCallback-5454-thread-2-processing-n:127.0.0.1:59471_solr) [n:127.0.0.1:59471_solr    ] o.a.s.c.LeaderElector 
   [junit4]   2> org.apache.solr.common.SolrException: SolrCore not found:implicitcoll_c_replica_n23 in []
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:297)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:56)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:348)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$3.lambda$process$0(SolrZkClient.java:269)
   [junit4]   2> 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 2471547 WARN  (zkCallback-5452-thread-2-processing-n:127.0.0.1:59473_solr) [n:127.0.0.1:59473_solr    ] o.a.s.c.LeaderElector Our node is no longer in line to be leader
   [junit4]   2> 2471547 WARN  (zkCallback-5455-thread-1-processing-n:127.0.0.1:59472_solr) [n:127.0.0.1:59472_solr    ] o.a.s.c.LeaderElector Our node is no longer in line to be leader
   [junit4]   2> 2471548 WARN  (zkCallback-5452-thread-1-processing-n:127.0.0.1:59473_solr) [n:127.0.0.1:59473_solr    ] o.a.s.c.LeaderElector 
   [junit4]   2> org.apache.solr.common.SolrException: SolrCore not found:implicitcoll_b_replica_n18 in []
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:297)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:56)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:348)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$3.lambda$process$0(SolrZkClient.java:269)
   [junit4]   2> 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 2471549 WARN  (zkCallback-5452-thread-1-processing-n:127.0.0.1:59473_solr) [n:127.0.0.1:59473_solr    ] o.a.s.c.LeaderElector 
   [junit4]   2> org.apache.solr.common.SolrException: SolrCore not found:implicitcoll_a_replica_n2 in []
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:297)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:56)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:348)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$3.lambda$process$0(SolrZkClient.java:269)
   [junit4]   2> 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 2471559 WARN  (zkCallback-5453-thread-2-processing-n:127.0.0.1:59470_solr) [n:127.0.0.1:59470_solr    ] o.a.s.c.LeaderElector Our node is no longer in line to be leader
   [junit4]   2> 2471560 WARN  (zkCallback-5452-thread-3-processing-n:127.0.0.1:59473_solr) [n:127.0.0.1:59473_solr    ] o.a.s.c.LeaderElector Our node is no longer in line to be leader
   [junit4]   2> 2471560 WARN  (zkCallback-5454-thread-2-processing-n:127.0.0.1:59471_solr) [n:127.0.0.1:59471_solr    ] o.a.s.c.LeaderElector Our node is no longer in line to be leader
   [junit4]   2> 2471561 WARN  (zkCallback-5453-thread-1-processing-n:127.0.0.1:59470_solr) [n:127.0.0.1:59470_solr    ] o.a.s.c.LeaderElector Our node is no longer in line to be leader
   [junit4]   2> 2502510 ERROR (qtp1174485167-23969) [n:127.0.0.1:59470_solr    ] o.a.s.h.a.CollectionsHandler Timed out waiting for new collection's replicas to become ACTIVE with timeout=30
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=CustomCollectionTest -Dtests.method=testCustomCollectionsAPI -Dtests.seed=688EE4121998BF56 -Dtests.slow=true -Dtests.locale=ar-OM -Dtests.timezone=Atlantic/Cape_Verde -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   45.7s J0 | CustomCollectionTest.testCustomCollectionsAPI <<<
   [junit4]    > Throwable #1: org.apache.solr.common.SolrException: Could not find collection : implicitcoll
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([688EE4121998BF56:26F6A792402092E]:0)
   [junit4]    > 	at org.apache.solr.common.cloud.ClusterState.getCollection(ClusterState.java:109)
   [junit4]    > 	at org.apache.solr.cloud.SolrCloudTestCase.getCollectionState(SolrCloudTestCase.java:247)
   [junit4]    > 	at org.apache.solr.cloud.CustomCollectionTest.testCustomCollectionsAPI(CustomCollectionTest.java:68)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 2504492 ERROR (qtp171749319-23939) [n:127.0.0.1:59472_solr c:withShardField s:a r:core_node5 x:withShardField_a_replica_n2] o.a.s.h.RequestHandlerBase org.apache.solr.common.SolrException: Error CREATEing SolrCore 'withShardField_a_replica_n2': Unable to create core [withShardField_a_replica_n2] Caused by: invalid API spec: apispec/core.config.Commands.json
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:971)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)
   [junit4]   2> 	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:745)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:726)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:507)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: Unable to create core [withShardField_a_replica_n2]
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1018)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:938)
   [junit4]   2> 	... 37 more
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: invalid API spec: apispec/core.config.Commands.json
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:988)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:843)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1002)
   [junit4]   2> 	... 38 more
   [junit4]   2> Caused by: java.lang.RuntimeException: invalid API spec: apispec/core.config.Commands.json
   [junit4]   2> 	at org.apache.solr.common.util.ValidatingJsonMap.parse(ValidatingJsonMap.java:318)
   [junit4]   2> 	at org.apache.solr.common.util.Utils.lambda$getSpec$0(Utils.java:394)
   [junit4]   2> 	at org.apache.solr.api.Api.getSpec(Api.java:65)
   [junit4]   2> 	at org.apache.solr.api.ApiBag.register(ApiBag.java:73)
   [junit4]   2> 	at org.apache.solr.core.PluginBag.put(PluginBag.java:217)
   [junit4]   2> 	at org.apache.solr.core.PluginBag.init(PluginBag.java:274)
   [junit4]   2> 	at org.apache.solr.core.RequestHandlers.initHandlersFromConfig(RequestHandlers.java:130)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:946)
   [junit4]   2> 	... 40 more
   [junit4]   2> 
   [junit4]   2> 2504492 ERROR (OverseerThreadFactory-10622-thread-5-processing-n:127.0.0.1:59473_solr) [n:127.0.0.1:59473_solr    ] o.a.s.c.OverseerCollectionMessageHandler Error from shard: https://127.0.0.1:59472/solr
   [junit4]   2> org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:59472/solr: Error CREATEing SolrCore 'withShardField_a_replica_n2': Unable to create core [withShardField_a_replica_n2] Caused by: invalid API spec: apispec/core.config.Commands.json
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:627)
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
   [junit4]   2> 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
   [junit4]   2> 	at org.apache.solr.handler.component.HttpShardHandler.lambda$submit$0(HttpShardHandler.java:172)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 2504493 ERROR (qtp1662558554-23961) [n:127.0.0.1:59473_solr c:withShardField s:b r:core_node11 x:withShardField_b_replica_n8] o.a.s.h.RequestHandlerBase org.apache.solr.common.SolrException: Error CREATEing SolrCore 'withShardField_b_replica_n8': Unable to create core [withShardField_b_replica_n8] Caused by: invalid API spec: apispec/core.config.Commands.json
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:971)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)
   [junit4]   2> 	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:745)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:726)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:507)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: Unable to create core [withShardField_b_replica_n8]
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1018)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:938)
   [junit4]   2> 	... 37 more
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: invalid API spec: apispec/core.config.Commands.json
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:988)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:843)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1002)
   [junit4]   2> 	... 38 more
   [junit4]   2> Caused by: java.lang.RuntimeException: invalid API spec: apispec/core.config.Commands.json
   [junit4]   2> 	at org.apache.solr.common.util.ValidatingJsonMap.parse(ValidatingJsonMap.java:318)
   [junit4]   2> 	at org.apache.solr.common.util.Utils.lambda$getSpec$0(Utils.java:394)
   [junit4]   2> 	at org.apache.solr.api.Api.getSpec(Api.java:65)
   [junit4]   2> 	at org.apache.solr.api.ApiBag.register(ApiBag.java:73)
   [junit4]   2> 	at org.apache.solr.core.PluginBag.put(PluginBag.java:217)
   [junit4]   2> 	at org.apache.solr.core.PluginBag.init(PluginBag.java:274)
   [junit4]   2> 	at org.apache.solr.core.RequestHandlers.initHandlersFromConfig(RequestHandlers.java:130)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:946)
   [junit4]   2> 	... 40 more
   [junit4]   2> 
   [junit4]   2> 2504494 ERROR (OverseerThreadFactory-10622-thread-5-processing-n:127.0.0.1:59473_solr) [n:127.0.0.1:59473_solr    ] o.a.s.c.OverseerCollectionMessageHandler Error from shard: https://127.0.0.1:59473/solr
   [junit4]   2> org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:59473/solr: Error CREATEing SolrCore 'withShardField_b_replica_n8': Unable to create core [withShardField_b_replica_n8] Caused by: invalid API spec: apispec/core.config.Commands.json
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:627)
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
   [junit4]   2> 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
   [junit4]   2> 	at org.apache.solr.handler.component.HttpShardHandler.lambda$submit$0(HttpShardHandler.java:172)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 2504496 ERROR (qtp655103699-23953) [n:127.0.0.1:59471_solr c:withShardField s:a r:core_node9 x:withShardField_a_replica_n6] o.a.s.h.RequestHandlerBase org.apache.solr.common.SolrException: Error CREATEing SolrCore 'withShardField_a_replica_n6': Unable to create core [withShardField_a_replica_n6] Caused by: invalid API spec: apispec/core.config.Commands.json
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:971)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)
   [junit4]   2> 	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:745)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:726)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:507)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   

[...truncated too long message...]

red for /live_nodes]
   [junit4]   2> 2537193 ERROR (SUITE-CustomCollectionTest-seed#[688EE4121998BF56]-worker) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 2537428 WARN  (Thread-4168) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	11	/solr/configs/conf
   [junit4]   2> 	9	/solr/aliases.json
   [junit4]   2> 	4	/solr/security.json
   [junit4]   2> 	4	/solr/collections/withShardField/state.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	49	/solr/collections/implicitcoll/state.json
   [junit4]   2> 	25	/solr/collections/testCreateShardRepFactor/state.json
   [junit4]   2> 	24	/solr/collections/routeFieldColl/state.json
   [junit4]   2> 	12	/solr/collections/withShardField/state.json
   [junit4]   2> 	9	/solr/clusterprops.json
   [junit4]   2> 	9	/solr/clusterstate.json
   [junit4]   2> 	3	/solr/overseer_elect/election/98566664058961929-127.0.0.1:59473_solr-n_0000000000
   [junit4]   2> 	2	/solr/collections/implicitcoll/leader_elect/b/election/98566664058961931-core_node19-n_0000000000
   [junit4]   2> 	2	/solr/collections/implicitcoll/leader_elect/a/election/98566664058961929-core_node5-n_0000000001
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	31	/solr/collections
   [junit4]   2> 	9	/solr/live_nodes
   [junit4]   2> 
   [junit4]   2> NOTE: leaving temporary files on disk at: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_688EE4121998BF56-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {shard_s=Lucene50(blocksize=128), _version_=Lucene50(blocksize=128), id=PostingsFormat(name=LuceneFixedGap)}, docValues:{_version_=DocValuesFormat(name=Lucene70)}, maxPointsInLeafNode=1986, maxMBSortInHeap=5.92319353753204, sim=RandomSimilarity(queryNorm=false): {}, locale=ar-OM, timezone=Atlantic/Cape_Verde
   [junit4]   2> NOTE: Mac OS X 10.11.6 x86_64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=3,threads=1,free=73699792,total=524288000
   [junit4]   2> NOTE: All tests run in this JVM: [TestNumericTerms32, TestSchemaResource, SolrMetricsIntegrationTest, TestChildDocTransformer, BasicAuthStandaloneTest, ClusterStateUpdateTest, InfixSuggestersTest, TestSolrConfigHandlerCloud, TestCoreContainer, TestTlogReplica, TestDistributedStatsComponentCardinality, BlockDirectoryTest, ChaosMonkeyNothingIsSafeTest, QueryElevationComponentTest, TestSubQueryTransformerCrossCore, DateMathParserTest, TestJsonFacetRefinement, DistribCursorPagingTest, TestReRankQParserPlugin, SolrIndexSplitterTest, SolrIndexMetricsTest, CursorPagingTest, SuggestComponentContextFilterQueryTest, MetricsConfigTest, BinaryUpdateRequestHandlerTest, RequestLoggingTest, SyncSliceTest, TestPayloadScoreQParserPlugin, BlockJoinFacetSimpleTest, TestMinMaxOnMultiValuedField, BigEndianAscendingWordDeserializerTest, OverriddenZkACLAndCredentialsProvidersTest, TestDefaultStatsCache, TestReversedWildcardFilterFactory, TestBinaryResponseWriter, DocValuesTest, TestStressReorder, ShardRoutingTest, FieldAnalysisRequestHandlerTest, TestRuleBasedAuthorizationPlugin, DistributedQueryComponentCustomSortTest, XsltUpdateRequestHandlerTest, PreAnalyzedUpdateProcessorTest, ReplicationFactorTest, HdfsChaosMonkeySafeLeaderTest, UniqFieldsUpdateProcessorFactoryTest, RollingRestartTest, TestDFRSimilarityFactory, TestRandomCollapseQParserPlugin, EnumFieldTest, TolerantUpdateProcessorTest, TestCorePropertiesReload, TestCloudPivotFacet, SolrCloudReportersTest, SortByFunctionTest, AnalysisAfterCoreReloadTest, SolrMetricManagerTest, ZkStateWriterTest, TestLazyCores, TestRebalanceLeaders, TestConfigsApi, TestBinaryField, TestIndexingPerformance, NotRequiredUniqueKeyTest, HdfsWriteToMultipleCollectionsTest, TestHighFrequencyDictionaryFactory, CoreAdminRequestStatusTest, TestCollationFieldDocValues, V2StandaloneTest, TestWriterPerf, TestManagedSchemaThreadSafety, TestPullReplica, SuggesterWFSTTest, TestUninvertingReader, TestQuerySenderNoQuery, MinimalSchemaTest, TestLockTree, TestDistribDocBasedVersion, TestFieldCacheWithThreads, TestDistributedSearch, DistributedSpellCheckComponentTest, CollectionTooManyReplicasTest, HighlighterConfigTest, TestPolicyCloud, TestRandomDVFaceting, DateRangeFieldTest, DefaultValueUpdateProcessorTest, TestFoldingMultitermQuery, DistributedSuggestComponentTest, TestSolrDeletionPolicy2, SolrCoreTest, QueryParsingTest, AssignBackwardCompatibilityTest, AtomicUpdatesTest, ChaosMonkeySafeLeaderWithPullReplicasTest, ZkControllerTest, TestRequestForwarding, TestInitQParser, TestXIncludeConfig, TestCloudRecovery, HighlighterTest, HdfsDirectoryTest, TestCryptoKeys, UUIDFieldTest, TestTrackingShardHandlerFactory, AsyncCallRequestStatusResponseTest, TestInfoStreamLogging, TestSolr4Spatial2, TestConfigOverlay, CdcrVersionReplicationTest, TestIntervalFaceting, HLLUtilTest, CloneFieldUpdateProcessorFactoryTest, TestSystemCollAutoCreate, SpatialRPTFieldTypeTest, SolrInfoBeanTest, TestFuzzyAnalyzedSuggestions, TestManagedSynonymFilterFactory, TestCodecSupport, RegexBoostProcessorTest, TestJmxIntegration, TestLegacyField, TestFieldCacheReopen, QueryResultKeyTest, DistanceUnitsTest, TestGroupingSearch, ForceLeaderTest, HttpSolrCallGetCoreTest, ClassificationUpdateProcessorIntegrationTest, MigrateRouteKeyTest, TestPerFieldSimilarityWithDefaultOverride, TestXmlQParser, TestSweetSpotSimilarityFactory, TestIBSimilarityFactory, TestPhraseSuggestions, LeaderFailoverAfterPartitionTest, CleanupOldIndexTest, TestSQLHandler, TestShardHandlerFactory, HdfsTlogReplayBufferedWhileIndexingTest, TestOmitPositions, ActionThrottleTest, DistributedFacetPivotLongTailTest, BitVectorTest, TestTolerantSearch, SimpleCollectionCreateDeleteTest, TestLeaderInitiatedRecoveryThread, TestCloudManagedSchema, TestAuthenticationFramework, UtilsToolTest, TestSolrCloudWithHadoopAuthPlugin, DeleteShardTest, TestSimpleQParserPlugin, SolrCloudExampleTest, TestMacroExpander, WrapperMergePolicyFactoryTest, RuleEngineTest, PeerSyncReplicationTest, MoveReplicaHDFSTest, SimplePostToolTest, ConfigSetsAPITest, PreAnalyzedFieldManagedSchemaCloudTest, TestQuerySenderListener, MetricUtilsTest, TestCursorMarkWithoutUniqueKey, ChangedSchemaMergeTest, LeaderElectionTest, TestManagedSchema, TestExtendedDismaxParser, SubstringBytesRefFilterTest, UnloadDistributedZkTest, SharedFSAutoReplicaFailoverTest, TestFieldResource, TestAtomicUpdateErrorCases, RequiredFieldsTest, TestObjectReleaseTracker, ZkFailoverTest, TestCollapseQParserPlugin, JSONWriterTest, BasicDistributedZk2Test, CollectionsAPIDistributedZkTest, OpenCloseCoreStressTest, OverseerTest, LeaderElectionIntegrationTest, BasicZkTest, RecoveryZkTest, FullSolrCloudDistribCmdsTest, TestReplicationHandler, TestRandomFaceting, ZkSolrClientTest, TestZkChroot, ZkCLITest, AutoCommitTest, TestFaceting, TestHashPartitioner, SimpleFacetsTest, PeerSyncTest, CoreAdminHandlerTest, SolrRequestParserTest, SuggesterTest, TestStressLucene, TestTrie, SpatialFilterTest, TestCSVLoader, PolyFieldTest, NoCacheHeaderTest, WordBreakSolrSpellCheckerTest, SchemaVersionSpecificBehaviorTest, SolrCoreCheckLockOnStartupTest, TestPseudoReturnFields, TestUpdate, FieldMutatingUpdateProcessorTest, DirectUpdateHandlerOptimizeTest, StatelessScriptUpdateProcessorFactoryTest, TestRemoteStreaming, DocValuesMultiTest, DistanceFunctionTest, DebugComponentTest, CacheHeaderTest, IndexBasedSpellCheckerTest, TestSurroundQueryParser, DisMaxRequestHandlerTest, DirectSolrSpellCheckerTest, TestQueryTypes, PrimitiveFieldTypeTest, FileBasedSpellCheckerTest, XmlUpdateRequestHandlerTest, RequestHandlersTest, TestValueSourceCache, PathHierarchyTokenizerFactoryTest, TermVectorComponentTest, MoreLikeThisHandlerTest, FastVectorHighlighterTest, SolrPluginUtilsTest, IndexSchemaTest, ReturnFieldsTest, UpdateRequestProcessorFactoryTest, TestCSVResponseWriter, TestPartialUpdateDeduplication, TestPHPSerializedResponseWriter, CSVRequestHandlerTest, SearchHandlerTest, TestLFUCache, MultiTermTest, SampleTest, TestDocSet, TestSearchPerf, NumericFieldsTest, TestElisionMultitermQuery, TestConfig, OutputWriterTest, TestSolrCoreProperties, DirectSolrConnectionTest, TestLuceneMatchVersion, SpellPossibilityIteratorTest, SynonymTokenizerTest, TestLMDirichletSimilarityFactory, TimeZoneUtilsTest, ResourceLoaderTest, ScriptEngineTest, OpenExchangeRatesOrgProviderTest, URLClassifyProcessorTest, PluginInfoTest, TestSystemIdResolver, PrimUtilsTest, DateFieldTest, RAMDirectoryFactoryTest, ClusterStateTest, TestLRUCache, TestUtils, ZkNodePropsTest, SliceStateTest, CircularListTest, DistributedMLTComponentTest, DistributedIntervalFacetingTest, SolrTestCaseJ4Test, TestHighlightDedupGrouping, TestEmbeddedSolrServerAdminHandler, TestEmbeddedSolrServerConstructors, TestEmbeddedSolrServerSchemaAPI, TestJettySolrRunner, BaseCdcrDistributedZkTest, CdcrBootstrapTest, ConnectionManagerTest, CreateCollectionCleanupTest, CustomCollectionTest]
   [junit4] Completed [504/732 (2!)] on J0 in 90.76s, 4 tests, 2 errors <<< FAILURES!

[...truncated 8992 lines...]
-validate-source-patterns:
[source-patterns] tabs instead spaces: lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoStandardPath.java

BUILD FAILED
/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/build.xml:810: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/build.xml:130: Found 1 violations in source files (tabs instead spaces).

Total time: 81 minutes 49 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any

[JENKINS] Lucene-Solr-master-MacOSX (64bit/jdk1.8.0) - Build # 4153 - Failure!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-MacOSX/4153/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseG1GC

1 tests failed.
FAILED:  org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.test

Error Message:
Could not find collection:collection2

Stack Trace:
java.lang.AssertionError: Could not find collection:collection2
	at __randomizedtesting.SeedInfo.seed([375EC011B19C0B54:BF0AFFCB1F6066AC]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNotNull(Assert.java:526)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:155)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:140)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:135)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:908)
	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.testIndexingBatchPerRequestWithHttpSolrClient(FullSolrCloudDistribCmdsTest.java:612)
	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.test(FullSolrCloudDistribCmdsTest.java:152)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 11277 lines...]
   [junit4] Suite: org.apache.solr.cloud.FullSolrCloudDistribCmdsTest
   [junit4]   2> 241000 INFO  (SUITE-FullSolrCloudDistribCmdsTest-seed#[375EC011B19C0B54]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_375EC011B19C0B54-001/init-core-data-001
   [junit4]   2> 241000 WARN  (SUITE-FullSolrCloudDistribCmdsTest-seed#[375EC011B19C0B54]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=6 numCloses=6
   [junit4]   2> 241000 INFO  (SUITE-FullSolrCloudDistribCmdsTest-seed#[375EC011B19C0B54]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 241004 INFO  (SUITE-FullSolrCloudDistribCmdsTest-seed#[375EC011B19C0B54]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776) w/ MAC_OS_X supressed clientAuth
   [junit4]   2> 241004 INFO  (SUITE-FullSolrCloudDistribCmdsTest-seed#[375EC011B19C0B54]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /qup/
   [junit4]   2> 241005 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 241005 INFO  (Thread-498) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 241005 INFO  (Thread-498) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 241008 ERROR (Thread-498) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 241108 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.c.ZkTestServer start zk server on port:61215
   [junit4]   2> 241155 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 241187 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
   [junit4]   2> 241190 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 241192 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 241194 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 241196 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 241198 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 241201 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 241201 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 241207 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 241210 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 241216 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
   [junit4]   2> 241458 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 241459 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@14c61b47{/qup,null,AVAILABLE}
   [junit4]   2> 241461 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@3e7da800{HTTP/1.1,[http/1.1]}{127.0.0.1:61222}
   [junit4]   2> 241461 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.e.j.s.Server Started @244396ms
   [junit4]   2> 241461 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_375EC011B19C0B54-001/tempDir-001/control/data, hostContext=/qup, hostPort=61222, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_375EC011B19C0B54-001/control-001/cores}
   [junit4]   2> 241461 ERROR (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 241462 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solrâ„¢ version 8.0.0
   [junit4]   2> 241462 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 241462 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 241462 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-08-29T00:49:22.728Z
   [junit4]   2> 241465 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 241465 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_375EC011B19C0B54-001/control-001/solr.xml
   [junit4]   2> 241470 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@76294c73, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 241473 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:61215/solr
   [junit4]   2> 241600 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [n:127.0.0.1:61222_qup    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 241600 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [n:127.0.0.1:61222_qup    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:61222_qup
   [junit4]   2> 241602 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [n:127.0.0.1:61222_qup    ] o.a.s.c.Overseer Overseer (id=98564031268388868-127.0.0.1:61222_qup-n_0000000000) starting
   [junit4]   2> 241613 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [n:127.0.0.1:61222_qup    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:61222_qup
   [junit4]   2> 241614 INFO  (zkCallback-468-thread-1-processing-n:127.0.0.1:61222_qup) [n:127.0.0.1:61222_qup    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 241673 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [n:127.0.0.1:61222_qup    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@76294c73
   [junit4]   2> 241679 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [n:127.0.0.1:61222_qup    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@76294c73
   [junit4]   2> 241679 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [n:127.0.0.1:61222_qup    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@76294c73
   [junit4]   2> 241680 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [n:127.0.0.1:61222_qup    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_375EC011B19C0B54-001/control-001/cores
   [junit4]   2> 241723 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 241724 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:61215/solr ready
   [junit4]   2> 241727 INFO  (qtp1362098068-2666) [n:127.0.0.1:61222_qup    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:61222_qup&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 241730 INFO  (OverseerThreadFactory-1483-thread-1-processing-n:127.0.0.1:61222_qup) [n:127.0.0.1:61222_qup    ] o.a.s.c.CreateCollectionCmd Create collection control_collection
   [junit4]   2> 241844 INFO  (qtp1362098068-2668) [n:127.0.0.1:61222_qup    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 241845 INFO  (qtp1362098068-2668) [n:127.0.0.1:61222_qup    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 241971 INFO  (zkCallback-468-thread-1-processing-n:127.0.0.1:61222_qup) [n:127.0.0.1:61222_qup    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 242866 INFO  (qtp1362098068-2668) [n:127.0.0.1:61222_qup c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 242875 INFO  (qtp1362098068-2668) [n:127.0.0.1:61222_qup c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
   [junit4]   2> 243131 INFO  (qtp1362098068-2668) [n:127.0.0.1:61222_qup c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 243170 INFO  (qtp1362098068-2668) [n:127.0.0.1:61222_qup c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
   [junit4]   2> 243171 INFO  (qtp1362098068-2668) [n:127.0.0.1:61222_qup c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@76294c73
   [junit4]   2> 243171 INFO  (qtp1362098068-2668) [n:127.0.0.1:61222_qup c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 243171 INFO  (qtp1362098068-2668) [n:127.0.0.1:61222_qup c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_375EC011B19C0B54-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_375EC011B19C0B54-001/control-001/cores/control_collection_shard1_replica_n1/data/]
   [junit4]   2> 243177 INFO  (qtp1362098068-2668) [n:127.0.0.1:61222_qup c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=31, maxMergeAtOnceExplicit=14, maxMergedSegmentMB=65.3056640625, floorSegmentMB=1.2509765625, forceMergeDeletesPctAllowed=11.64130481790744, segmentsPerTier=17.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 243187 WARN  (qtp1362098068-2668) [n:127.0.0.1:61222_qup c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 243251 INFO  (qtp1362098068-2668) [n:127.0.0.1:61222_qup c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 243251 INFO  (qtp1362098068-2668) [n:127.0.0.1:61222_qup c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 243253 INFO  (qtp1362098068-2668) [n:127.0.0.1:61222_qup c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 243253 INFO  (qtp1362098068-2668) [n:127.0.0.1:61222_qup c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 243276 INFO  (qtp1362098068-2668) [n:127.0.0.1:61222_qup c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=17, maxMergeAtOnceExplicit=19, maxMergedSegmentMB=83.12890625, floorSegmentMB=0.98828125, forceMergeDeletesPctAllowed=20.30844128165794, segmentsPerTier=20.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 243276 INFO  (qtp1362098068-2668) [n:127.0.0.1:61222_qup c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@47d141dc[control_collection_shard1_replica_n1] main]
   [junit4]   2> 243278 INFO  (qtp1362098068-2668) [n:127.0.0.1:61222_qup c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 243282 INFO  (qtp1362098068-2668) [n:127.0.0.1:61222_qup c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 243283 INFO  (qtp1362098068-2668) [n:127.0.0.1:61222_qup c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 243284 INFO  (searcherExecutor-1486-thread-1-processing-n:127.0.0.1:61222_qup x:control_collection_shard1_replica_n1 s:shard1 c:control_collection) [n:127.0.0.1:61222_qup c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@47d141dc[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 243285 INFO  (qtp1362098068-2668) [n:127.0.0.1:61222_qup c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1577024502681829376
   [junit4]   2> 243295 INFO  (qtp1362098068-2668) [n:127.0.0.1:61222_qup c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 243296 INFO  (qtp1362098068-2668) [n:127.0.0.1:61222_qup c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 243296 INFO  (qtp1362098068-2668) [n:127.0.0.1:61222_qup c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:61222/qup/control_collection_shard1_replica_n1/
   [junit4]   2> 243296 INFO  (qtp1362098068-2668) [n:127.0.0.1:61222_qup c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 243296 INFO  (qtp1362098068-2668) [n:127.0.0.1:61222_qup c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:61222/qup/control_collection_shard1_replica_n1/ has no replicas
   [junit4]   2> 243296 INFO  (qtp1362098068-2668) [n:127.0.0.1:61222_qup c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 243302 INFO  (qtp1362098068-2668) [n:127.0.0.1:61222_qup c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:61222/qup/control_collection_shard1_replica_n1/ shard1
   [junit4]   2> 243405 INFO  (zkCallback-468-thread-1-processing-n:127.0.0.1:61222_qup) [n:127.0.0.1:61222_qup    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 243457 INFO  (qtp1362098068-2668) [n:127.0.0.1:61222_qup c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 243459 INFO  (qtp1362098068-2668) [n:127.0.0.1:61222_qup c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1614
   [junit4]   2> 243462 INFO  (qtp1362098068-2666) [n:127.0.0.1:61222_qup    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 243565 INFO  (zkCallback-468-thread-1-processing-n:127.0.0.1:61222_qup) [n:127.0.0.1:61222_qup    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 243734 INFO  (OverseerCollectionConfigSetProcessor-98564031268388868-127.0.0.1:61222_qup-n_0000000000) [n:127.0.0.1:61222_qup    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 244465 INFO  (qtp1362098068-2666) [n:127.0.0.1:61222_qup    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:61222_qup&wt=javabin&version=2} status=0 QTime=2737
   [junit4]   2> 244479 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 244480 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:61215/solr ready
   [junit4]   2> 244480 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 244481 INFO  (qtp1362098068-2671) [n:127.0.0.1:61222_qup    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=3&createNodeSet=&stateFormat=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 244484 INFO  (OverseerThreadFactory-1483-thread-2-processing-n:127.0.0.1:61222_qup) [n:127.0.0.1:61222_qup    ] o.a.s.c.CreateCollectionCmd Create collection collection1
   [junit4]   2> 244485 WARN  (OverseerThreadFactory-1483-thread-2-processing-n:127.0.0.1:61222_qup) [n:127.0.0.1:61222_qup    ] o.a.s.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
   [junit4]   2> 244693 INFO  (qtp1362098068-2671) [n:127.0.0.1:61222_qup    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 244693 INFO  (qtp1362098068-2671) [n:127.0.0.1:61222_qup    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=3&createNodeSet=&stateFormat=1&wt=javabin&version=2} status=0 QTime=211
   [junit4]   2> 245246 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_375EC011B19C0B54-001/shard-1-001 of type NRT
   [junit4]   2> 245250 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 245252 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6f340b3b{/qup,null,AVAILABLE}
   [junit4]   2> 245253 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@1f4dff9e{HTTP/1.1,[http/1.1]}{127.0.0.1:61246}
   [junit4]   2> 245254 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.e.j.s.Server Started @248188ms
   [junit4]   2> 245254 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_375EC011B19C0B54-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/qup, hostPort=61246, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_375EC011B19C0B54-001/shard-1-001/cores}
   [junit4]   2> 245256 ERROR (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 245256 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solrâ„¢ version 8.0.0
   [junit4]   2> 245256 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 245256 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 245256 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-08-29T00:49:26.522Z
   [junit4]   2> 245259 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 245259 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_375EC011B19C0B54-001/shard-1-001/solr.xml
   [junit4]   2> 245265 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@76294c73, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 245270 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:61215/solr
   [junit4]   2> 245282 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [n:127.0.0.1:61246_qup    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 245286 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [n:127.0.0.1:61246_qup    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 245288 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [n:127.0.0.1:61246_qup    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:61246_qup
   [junit4]   2> 245291 INFO  (zkCallback-468-thread-1-processing-n:127.0.0.1:61222_qup) [n:127.0.0.1:61222_qup    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 245291 INFO  (zkCallback-475-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 245291 INFO  (zkCallback-480-thread-1-processing-n:127.0.0.1:61246_qup) [n:127.0.0.1:61246_qup    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 245390 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [n:127.0.0.1:61246_qup    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@76294c73
   [junit4]   2> 245398 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [n:127.0.0.1:61246_qup    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@76294c73
   [junit4]   2> 245398 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [n:127.0.0.1:61246_qup    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@76294c73
   [junit4]   2> 245399 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [n:127.0.0.1:61246_qup    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_375EC011B19C0B54-001/shard-1-001/cores
   [junit4]   2> 245456 INFO  (qtp1362098068-2664) [n:127.0.0.1:61222_qup    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:61246_qup&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 245459 INFO  (OverseerThreadFactory-1483-thread-3-processing-n:127.0.0.1:61222_qup) [n:127.0.0.1:61222_qup    ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:61246_qup for creating new replica
   [junit4]   2> 245460 INFO  (OverseerCollectionConfigSetProcessor-98564031268388868-127.0.0.1:61222_qup-n_0000000000) [n:127.0.0.1:61222_qup    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 245467 INFO  (qtp162240256-2707) [n:127.0.0.1:61246_qup    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n61&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 245467 INFO  (qtp162240256-2707) [n:127.0.0.1:61246_qup    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 246526 INFO  (qtp162240256-2707) [n:127.0.0.1:61246_qup c:collection1 s:shard2  x:collection1_shard2_replica_n61] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 246538 INFO  (qtp162240256-2707) [n:127.0.0.1:61246_qup c:collection1 s:shard2  x:collection1_shard2_replica_n61] o.a.s.s.IndexSchema [collection1_shard2_replica_n61] Schema name=test
   [junit4]   2> 246729 INFO  (qtp162240256-2707) [n:127.0.0.1:61246_qup c:collection1 s:shard2  x:collection1_shard2_replica_n61] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 246746 INFO  (qtp162240256-2707) [n:127.0.0.1:61246_qup c:collection1 s:shard2  x:collection1_shard2_replica_n61] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_n61' using configuration from collection collection1, trusted=true
   [junit4]   2> 246747 INFO  (qtp162240256-2707) [n:127.0.0.1:61246_qup c:collection1 s:shard2  x:collection1_shard2_replica_n61] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_n61' (registry 'solr.core.collection1.shard2.replica_n61') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@76294c73
   [junit4]   2> 246747 INFO  (qtp162240256-2707) [n:127.0.0.1:61246_qup c:collection1 s:shard2  x:collection1_shard2_replica_n61] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 246747 INFO  (qtp162240256-2707) [n:127.0.0.1:61246_qup c:collection1 s:shard2  x:collection1_shard2_replica_n61] o.a.s.c.SolrCore [[collection1_shard2_replica_n61] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_375EC011B19C0B54-001/shard-1-001/cores/collection1_shard2_replica_n61], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_375EC011B19C0B54-001/shard-1-001/cores/collection1_shard2_replica_n61/data/]
   [junit4]   2> 246751 INFO  (qtp162240256-2707) [n:127.0.0.1:61246_qup c:collection1 s:shard2  x:collection1_shard2_replica_n61] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=31, maxMergeAtOnceExplicit=14, maxMergedSegmentMB=65.3056640625, floorSegmentMB=1.2509765625, forceMergeDeletesPctAllowed=11.64130481790744, segmentsPerTier=17.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 246773 WARN  (qtp162240256-2707) [n:127.0.0.1:61246_qup c:collection1 s:shard2  x:collection1_shard2_replica_n61] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 246835 INFO  (qtp162240256-2707) [n:127.0.0.1:61246_qup c:collection1 s:shard2  x:collection1_shard2_replica_n61] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 246835 INFO  (qtp162240256-2707) [n:127.0.0.1:61246_qup c:collection1 s:shard2  x:collection1_shard2_replica_n61] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 246837 INFO  (qtp162240256-2707) [n:127.0.0.1:61246_qup c:collection1 s:shard2  x:collection1_shard2_replica_n61] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 246837 INFO  (qtp162240256-2707) [n:127.0.0.1:61246_qup c:collection1 s:shard2  x:collection1_shard2_replica_n61] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 246839 INFO  (qtp162240256-2707) [n:127.0.0.1:61246_qup c:collection1 s:shard2  x:collection1_shard2_replica_n61] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=17, maxMergeAtOnceExplicit=19, maxMergedSegmentMB=83.12890625, floorSegmentMB=0.98828125, forceMergeDeletesPctAllowed=20.30844128165794, segmentsPerTier=20.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 246846 INFO  (qtp162240256-2707) [n:127.0.0.1:61246_qup c:collection1 s:shard2  x:collection1_shard2_replica_n61] o.a.s.s.SolrIndexSearcher Opening [Searcher@27ce34a2[collection1_shard2_replica_n61] main]
   [junit4]   2> 246847 INFO  (qtp162240256-2707) [n:127.0.0.1:61246_qup c:collection1 s:shard2  x:collection1_shard2_replica_n61] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 246848 INFO  (qtp162240256-2707) [n:127.0.0.1:61246_qup c:collection1 s:shard2  x:collection1_shard2_replica_n61] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 246849 INFO  (qtp162240256-2707) [n:127.0.0.1:61246_qup c:collection1 s:shard2  x:collection1_shard2_replica_n61] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 246849 INFO  (qtp162240256-2707) [n:127.0.0.1:61246_qup c:collection1 s:shard2  x:collection1_shard2_replica_n61] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1577024506418954240
   [junit4]   2> 246849 INFO  (searcherExecutor-1497-thread-1-processing-n:127.0.0.1:61246_qup x:collection1_shard2_replica_n61 s:shard2 c:collection1) [n:127.0.0.1:61246_qup c:collection1 s:shard2  x:collection1_shard2_replica_n61] o.a.s.c.SolrCore [collection1_shard2_replica_n61] Registered new searcher Searcher@27ce34a2[collection1_shard2_replica_n61] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 246860 INFO  (qtp162240256-2707) [n:127.0.0.1:61246_qup c:collection1 s:shard2  x:collection1_shard2_replica_n61] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 246860 INFO  (qtp162240256-2707) [n:127.0.0.1:61246_qup c:collection1 s:shard2  x:collection1_shard2_replica_n61] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 246860 INFO  (qtp162240256-2707) [n:127.0.0.1:61246_qup c:collection1 s:shard2  x:collection1_shard2_replica_n61] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:61246/qup/collection1_shard2_replica_n61/
   [junit4]   2> 246860 INFO  (qtp162240256-2707) [n:127.0.0.1:61246_qup c:collection1 s:shard2  x:collection1_shard2_replica_n61] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 246861 INFO  (qtp162240256-2707) [n:127.0.0.1:61246_qup c:collection1 s:shard2  x:collection1_shard2_replica_n61] o.a.s.c.SyncStrategy http://127.0.0.1:61246/qup/collection1_shard2_replica_n61/ has no replicas
   [junit4]   2> 246861 INFO  (qtp162240256-2707) [n:127.0.0.1:61246_qup c:collection1 s:shard2  x:collection1_shard2_replica_n61] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 246866 INFO  (qtp162240256-2707) [n:127.0.0.1:61246_qup c:collection1 s:shard2  x:collection1_shard2_replica_n61] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:61246/qup/collection1_shard2_replica_n61/ shard2
   [junit4]   2> 246976 INFO  (qtp162240256-2707) [n:127.0.0.1:61246_qup c:collection1 s:shard2  x:collection1_shard2_replica_n61] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 246977 INFO  (qtp162240256-2707) [n:127.0.0.1:61246_qup c:collection1 s:shard2  x:collection1_shard2_replica_n61] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n61&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1510
   [junit4]   2> 246980 INFO  (qtp1362098068-2664) [n:127.0.0.1:61222_qup    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:61246_qup&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2} status=0 QTime=1523
   [junit4]   2> 247468 INFO  (OverseerCollectionConfigSetProcessor-98564031268388868-127.0.0.1:61222_qup-n_0000000000) [n:127.0.0.1:61222_qup    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 247684 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_375EC011B19C0B54-001/shard-2-001 of type NRT
   [junit4]   2> 247684 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 247686 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@78d225bc{/qup,null,AVAILABLE}
   [junit4]   2> 247687 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@530a5c25{HTTP/1.1,[http/1.1]}{127.0.0.1:61263}
   [junit4]   2> 247687 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.e.j.s.Server Started @250621ms
   [junit4]   2> 247687 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_375EC011B19C0B54-001/tempDir-001/jetty2, solrconfig=solrconfig.xml, hostContext=/qup, hostPort=61263, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_375EC011B19C0B54-001/shard-2-001/cores}
   [junit4]   2> 247687 ERROR (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 247689 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solrâ„¢ version 8.0.0
   [junit4]   2> 247689 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 247690 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 247690 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-08-29T00:49:28.956Z
   [junit4]   2> 247732 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 247732 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_375EC011B19C0B54-001/shard-2-001/solr.xml
   [junit4]   2> 247761 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@76294c73, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 247765 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:61215/solr
   [junit4]   2> 247800 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [n:127.0.0.1:61263_qup    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 247804 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [n:127.0.0.1:61263_qup    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 247806 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [n:127.0.0.1:61263_qup    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:61263_qup
   [junit4]   2> 247808 INFO  (zkCallback-468-thread-1-processing-n:127.0.0.1:61222_qup) [n:127.0.0.1:61222_qup    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 247809 INFO  (zkCallback-486-thread-1-processing-n:127.0.0.1:61263_qup) [n:127.0.0.1:61263_qup    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 247808 INFO  (zkCallback-475-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 247808 INFO  (zkCallback-480-thread-1-processing-n:127.0.0.1:61246_qup) [n:127.0.0.1:61246_qup    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 248111 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [n:127.0.0.1:61263_qup    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@76294c73
   [junit4]   2> 248122 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [n:127.0.0.1:61263_qup    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@76294c73
   [junit4]   2> 248122 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [n:127.0.0.1:61263_qup    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@76294c73
   [junit4]   2> 248125 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [n:127.0.0.1:61263_qup    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_375EC011B19C0B54-001/shard-2-001/cores
   [junit4]   2> 248168 INFO  (qtp1533496363-2736) [n:127.0.0.1:61263_qup    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:61263_qup&action=ADDREPLICA&collection=collection1&shard=shard3&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 248171 INFO  (OverseerThreadFactory-1483-thread-4-processing-n:127.0.0.1:61222_qup) [n:127.0.0.1:61222_qup    ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:61263_qup for creating new replica
   [junit4]   2> 248174 INFO  (qtp1533496363-2738) [n:127.0.0.1:61263_qup    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard3_replica_n63&action=CREATE&collection=collection1&shard=shard3&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 248175 INFO  (qtp1533496363-2738) [n:127.0.0.1:61263_qup    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 249193 INFO  (qtp1533496363-2738) [n:127.0.0.1:61263_qup c:collection1 s:shard3  x:collection1_shard3_replica_n63] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 249202 INFO  (qtp1533496363-2738) [n:127.0.0.1:61263_qup c:collection1 s:shard3  x:collection1_shard3_replica_n63] o.a.s.s.IndexSchema [collection1_shard3_replica_n63] Schema name=test
   [junit4]   2> 249290 INFO  (qtp1533496363-2738) [n:127.0.0.1:61263_qup c:collection1 s:shard3  x:collection1_shard3_replica_n63] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 249319 INFO  (qtp1533496363-2738) [n:127.0.0.1:61263_qup c:collection1 s:shard3  x:collection1_shard3_replica_n63] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard3_replica_n63' using configuration from collection collection1, trusted=true
   [junit4]   2> 249320 INFO  (qtp1533496363-2738) [n:127.0.0.1:61263_qup c:collection1 s:shard3  x:collection1_shard3_replica_n63] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard3.replica_n63' (registry 'solr.core.collection1.shard3.replica_n63') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@76294c73
   [junit4]   2> 249320 INFO  (qtp1533496363-2738) [n:127.0.0.1:61263_qup c:collection1 s:shard3  x:collection1_shard3_replica_n63] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 249320 INFO  (qtp1533496363-2738) [n:127.0.0.1:61263_qup c:collection1 s:shard3  x:collection1_shard3_replica_n63] o.a.s.c.SolrCore [[collection1_shard3_replica_n63] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_375EC011B19C0B54-001/shard-2-001/cores/collection1_shard3_replica_n63], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_375EC011B19C0B54-001/shard-2-001/cores/collection1_shard3_replica_n63/data/]
   [junit4]   2> 249327 INFO  (qtp1533496363-2738) [n:127.0.0.1:61263_qup c:collection1 s:shard3  x:collection1_shard3_replica_n63] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=31, maxMergeAtOnceExplicit=14, maxMergedSegmentMB=65.3056640625, floorSegmentMB=1.2509765625, forceMergeDeletesPctAllowed=11.64130481790744, segmentsPerTier=17.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 249340 WARN  (qtp1533496363-2738) [n:127.0.0.1:61263_qup c:collection1 s:shard3  x:collection1_shard3_replica_n63] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 249408 INFO  (qtp1533496363-2738) [n:127.0.0.1:61263_qup c:collection1 s:shard3  x:collection1_shard3_replica_n63] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 249408 INFO  (qtp1533496363-2738) [n:127.0.0.1:61263_qup c:collection1 s:shard3  x:collection1_shard3_replica_n63] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 249419 INFO  (qtp1533496363-2738) [n:127.0.0.1:61263_qup c:collection1 s:shard3  x:collection1_shard3_replica_n63] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 249419 INFO  (qtp1533496363-2738) [n:127.0.0.1:61263_qup c:collection1 s:shard3  x:collection1_shard3_replica_n63] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 249435 INFO  (qtp1533496363-2738) [n:127.0.0.1:61263_qup c:collection1 s:shard3  x:collection1_shard3_replica_n63] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=17, maxMergeAtOnceExplicit=19, maxMergedSegmentMB=83.12890625, floorSegmentMB=0.98828125, forceMergeDeletesPctAllowed=20.30844128165794, segmentsPerTier=20.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 249447 INFO  (qtp1533496363-2738) [n:127.0.0.1:61263_qup c:collection1 s:shard3  x:collection1_shard3_replica_n63] o.a.s.s.SolrIndexSearcher Opening [Searcher@3423fdfe[collection1_shard3_replica_n63] main]
   [junit4]   2> 249449 INFO  (qtp1533496363-2738) [n:127.0.0.1:61263_qup c:collection1 s:shard3  x:collection1_shard3_replica_n63] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 249450 INFO  (qtp1533496363-2738) [n:127.0.0.1:61263_qup c:collection1 s:shard3  x:collection1_shard3_replica_n63] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 249452 INFO  (qtp1533496363-2738) [n:127.0.0.1:61263_qup c:collection1 s:shard3  x:collection1_shard3_replica_n63] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 249453 INFO  (searcherExecutor-1508-thread-1-processing-n:127.0.0.1:61263_qup x:collection1_shard3_replica_n63 s:shard3 c:collection1) [n:127.0.0.1:61263_qup c:collection1 s:shard3  x:collection1_shard3_replica_n63] o.a.s.c.SolrCore [collection1_shard3_replica_n63] Registered new searcher Searcher@3423fdfe[collection1_shard3_replica_n63] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 249453 INFO  (qtp1533496363-2738) [n:127.0.0.1:61263_qup c:collection1 s:shard3  x:collection1_shard3_replica_n63] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1577024509149446144
   [junit4]   2> 249470 INFO  (qtp1533496363-2738) [n:127.0.0.1:61263_qup c:collection1 s:shard3  x:collection1_shard3_replica_n63] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 249471 INFO  (qtp1533496363-2738) [n:127.0.0.1:61263_qup c:collection1 s:shard3  x:collection1_shard3_replica_n63] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 249471 INFO  (qtp1533496363-2738) [n:127.0.0.1:61263_qup c:collection1 s:shard3  x:collection1_shard3_replica_n63] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:61263/qup/collection1_shard3_replica_n63/
   [junit4]   2> 249471 INFO  (qtp1533496363-2738) [n:127.0.0.1:61263_qup c:collection1 s:shard3  x:collection1_shard3_replica_n63] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 249471 INFO  (qtp1533496363-2738) [n:127.0.0.1:61263_qup c:collection1 s:shard3  x:collection1_shard3_replica_n63] o.a.s.c.SyncStrategy http://127.0.0.1:61263/qup/collection1_shard3_replica_n63/ has no replicas
   [junit4]   2> 249471 INFO  (qtp1533496363-2738) [n:127.0.0.1:61263_qup c:collection1 s:shard3  x:collection1_shard3_replica_n63] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 249476 INFO  (qtp1533496363-2738) [n:127.0.0.1:61263_qup c:collection1 s:shard3  x:collection1_shard3_replica_n63] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:61263/qup/collection1_shard3_replica_n63/ shard3
   [junit4]   2> 249586 INFO  (qtp1533496363-2738) [n:127.0.0.1:61263_qup c:collection1 s:shard3  x:collection1_shard3_replica_n63] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 249588 INFO  (qtp1533496363-2738) [n:127.0.0.1:61263_qup c:collection1 s:shard3  x:collection1_shard3_replica_n63] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard3_replica_n63&action=CREATE&collection=collection1&shard=shard3&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1413
   [junit4]   2> 249591 INFO  (qtp1533496363-2736) [n:127.0.0.1:61263_qup    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:61263_qup&action=ADDREPLICA&collection=collection1&shard=shard3&type=NRT&wt=javabin&version=2} status=0 QTime=1422
   [junit4]   2> 250054 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_375EC011B19C0B54-001/shard-3-001 of type NRT
   [junit4]   2> 250055 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 250057 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@38d4fb52{/qup,null,AVAILABLE}
   [junit4]   2> 250059 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@6d89575e{HTTP/1.1,[http/1.1]}{127.0.0.1:61285}
   [junit4]   2> 250059 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.e.j.s.Server Started @252993ms
   [junit4]   2> 250059 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_375EC011B19C0B54-001/tempDir-001/jetty3, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/qup, hostPort=61285, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_375EC011B19C0B54-001/shard-3-001/cores}
   [junit4]   2> 250059 ERROR (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 250060 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solrâ„¢ version 8.0.0
   [junit4]   2> 250062 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 250062 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 250062 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-08-29T00:49:31.328Z
   [junit4]   2> 250065 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 250065 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_375EC011B19C0B54-001/shard-3-001/solr.xml
   [junit4]   2> 250071 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@76294c73, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 250111 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:61215/solr
   [junit4]   2> 250124 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [n:127.0.0.1:61285_qup    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
   [junit4]   2> 250127 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [n:127.0.0.1:61285_qup    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 250129 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [n:127.0.0.1:61285_qup    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:61285_qup
   [junit4]   2> 250131 INFO  (zkCallback-468-thread-2-processing-n:127.0.0.1:61222_qup) [n:127.0.0.1:61222_qup    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 250132 INFO  (zkCallback-480-thread-1-processing-n:127.0.0.1:61246_qup) [n:127.0.0.1:61246_qup    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 250132 INFO  (zkCallback-475-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 250132 INFO  (zkCallback-486-thread-1-processing-n:127.0.0.1:61263_qup) [n:127.0.0.1:61263_qup    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 250133 INFO  (zkCallback-492-thread-1-processing-n:127.0.0.1:61285_qup) [n:127.0.0.1:61285_qup    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 250173 INFO  (OverseerCollectionConfigSetProcessor-98564031268388868-127.0.0.1:61222_qup-n_0000000000) [n:127.0.0.1:61222_qup    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000006 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 250237 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [n:127.0.0.1:61285_qup    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@76294c73
   [junit4]   2> 250253 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [n:127.0.0.1:61285_qup    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@76294c73
   [junit4]   2> 250253 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [n:127.0.0.1:61285_qup    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@76294c73
   [junit4]   2> 250255 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [n:127.0.0.1:61285_qup    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_375EC011B19C0B54-001/shard-3-001/cores
   [junit4]   2> 250305 INFO  (qtp1533496363-2739) [n:127.0.0.1:61263_qup    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:61285_qup&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 250308 INFO  (OverseerThreadFactory-1483-thread-5-processing-n:127.0.0.1:61222_qup) [n:127.0.0.1:61222_qup    ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:61285_qup for creating new replica
   [junit4]   2> 250313 INFO  (qtp777203105-2766) [n:127.0.0.1:61285_qup    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n65&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 250313 INFO  (qtp777203105-2766) [n:127.0.0.1:61285_qup    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 251330 INFO  (qtp777203105-2766) [n:127.0.0.1:61285_qup c:collection1 s:shard1  x:collection1_shard1_replica_n65] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 251339 INFO  (qtp777203105-2766) [n:127.0.0.1:61285_qup c:collection1 s:shard1  x:collection1_shard1_replica_n65] o.a.s.s.IndexSchema [collection1_shard1_replica_n65] Schema name=test
   [junit4]   2> 251441 INFO  (qtp777203105-2766) [n:127.0.0.1:61285_qup c:collection1 s:shard1  x:collection1_shard1_replica_n65] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 251479 INFO  (qtp777203105-2766) [n:127.0.0.1:61285_qup c:collection1 s:shard1  x:collection1_shard1_replica_n65] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_n65' using configuration from collection collection1, trusted=true
   [junit4]   2> 251480 INFO  (qtp777203105-2766) [n:127.0.0.1:61285_qup c:collection1 s:shard1  x:collection1_shard1_replica_n65] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.replica_n65' (registry 'solr.core.collection1.shard1.replica_n65') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@76294c73
   [junit4]   2> 251480 INFO  (qtp777203105-2766) [n:127.0.0.1:61285_qup c:collection1 s:shard1  x:collection1_shard1_replica_n65] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 251480 INFO  (qtp777203105-2766) [n:127.0.0.1:61285_qup c:collection1 s:shard1  x:collection1_shard1_replica_n65] o.a.s.c.SolrCore [[collection1_shard1_replica_n65] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_375EC011B19C0B54-001/shard-3-001/cores/collection1_shard1_replica_n65], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_375EC011B19C0B54-001/shard-3-001/cores/collection1_shard1_replica_n65/data/]
   [junit4]   2> 251493 INFO  (qtp777203105-2766) [n:127.0.0.1:61285_qup c:collection1 s:shard1  x:collection1_shard1_replica_n65] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=31, maxMergeAtOnceExplicit=14, maxMergedSegmentMB=65.3056640625, floorSegmentMB=1.2509765625, forceMergeDeletesPctAllowed=11.64130481790744, segmentsPerTier=17.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 251527 WARN  (qtp777203105-2766) [n:127.0.0.1:61285_qup c:collection1 s:shard1  x:collection1_shard1_replica_n65] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 251576 INFO  (qtp777203105-2766) [n:127.0.0.1:61285_qup c:collection1 s:shard1  x:collection1_shard1_replica_n65] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 251576 INFO  (qtp777203105-2766) [n:127.0.0.1:61285_qup c:collection1 s:shard1  x:collection1_shard1_replica_n65] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 251578 INFO  (qtp777203105-2766) [n:127.0.0.1:61285_qup c:collection1 s:shard1  x:collection1_shard1_replica_n65] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 251578 INFO  (qtp777203105-2766) [n:127.0.0.1:61285_qup c:collection1 s:shard1  x:collection1_shard1_replica_n65] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 251579 INFO  (qtp777203105-2766) [n:127.0.0.1:61285_qup c:collection1 s:shard1  x:collection1_shard1_replica_n65] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=17, maxMergeAtOnceExplicit=19, maxMergedSegmentMB=83.12890625, floorSegmentMB=0.98828125, forceMergeDeletesPctAllowed=20.30844128165794, segmentsPerTier=20.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 251579 INFO  (qtp777203105-2766) [n:127.0.0.1:61285_qup c:collection1 s:shard1  x:collection1_shard1_replica_n65] o.a.s.s.SolrIndexSearcher Opening [Searcher@6357ed39[collection1_shard1_replica_n65] main]
   [junit4]   2> 251581 INFO  (qtp777203105-2766) [n:127.0.0.1:61285_qup c:collection1 s:shard1  x:collection1_shard1_replica_n65] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 251581 INFO  (qtp777203105-2766) [n:127.0.0.1:61285_qup c:collection1 s:shard1  x:collection1_shard1_replica_n65] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 251582 INFO  (qtp777203105-2766) [n:127.0.0.1:61285_qup c:collection1 s:shard1  x:collection1_shard1_replica_n65] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 251582 INFO  (qtp777203105-2766) [n:127.0.0.1:61285_qup c:collection1 s:shard1  x:collection1_shard1_replica_n65] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1577024511381864448
   [junit4]   2> 251583 INFO  (searcherExecutor-1519-thread-1-processing-n:127.0.0.1:61285_qup x:collection1_shard1_replica_n65 s:shard1 c:collection1) [n:127.0.0.1:61285_qup c:collection1 s:shard1  x:collection1_shard1_replica_n65] o.a.s.c.SolrCore [collection1_shard1_replica_n65] Registered new searcher Searcher@6357ed39[collection1_shard1_replica_n65] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 251588 INFO  (qtp777203105-2766) [n:127.0.0.1:61285_qup c:collection1 s:shard1  x:collection1_shard1_replica_n65] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 251588 INFO  (qtp777203105-2766) [n:127.0.0.1:61285_qup c:collection1 s:shard1  x:collection1_shard1_replica_n65] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 251588 INFO  (qtp777203105-2766) [n:127.0.0.1:61285_qup c:collection1 s:shard1  x:collection1_shard1_replica_n65] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:61285/qup/collection1_shard1_replica_n65/
   [junit4]   2> 251589 INFO  (qtp777203105-2766) [n:127.0.0.1:61285_qup c:collection1 s:shard1  x:collection1_shard1_replica_n65] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 251589 INFO  (qtp777203105-2766) [n:127.0.0.1:61285_qup c:collection1 s:shard1  x:collection1_shard1_replica_n65] o.a.s.c.SyncStrategy http://127.0.0.1:61285/qup/collection1_shard1_replica_n65/ has no replicas
   [junit4]   2> 251589 INFO  (qtp777203105-2766) [n:127.0.0.1:61285_qup c:collection1 s:shard1  x:collection1_shard1_replica_n65] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 251593 INFO  (qtp777203105-2766) [n:127.0.0.1:61285_qup c:collection1 s:shard1  x:collection1_shard1_replica_n65] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:61285/qup/collection1_shard1_replica_n65/ shard1
   [junit4]   2> 251763 INFO  (qtp777203105-2766) [n:127.0.0.1:61285_qup c:collection1 s:shard1  x:collection1_shard1_replica_n65] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 251765 INFO  (qtp777203105-2766) [n:127.0.0.1:61285_qup c:collection1 s:shard1  x:collection1_shard1_replica_n65] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n65&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1451
   [junit4]   2> 251768 INFO  (qtp1533496363-2739) [n:127.0.0.1:61263_qup    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:61285_qup&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2} status=0 QTime=1462
   [junit4]   2> 252271 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 4 in directory /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_375EC011B19C0B54-001/shard-4-001 of type NRT
   [junit4]   2> 252271 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 252273 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1c608de4{/qup,null,AVAILABLE}
   [junit4]   2> 252274 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@5457caf5{HTTP/1.1,[http/1.1]}{127.0.0.1:61302}
   [junit4]   2> 252274 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.e.j.s.Server Started @255208ms
   [junit4]   2> 252274 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_375EC011B19C0B54-001/tempDir-001/jetty4, solrconfig=solrconfig.xml, hostContext=/qup, hostPort=61302, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_375EC011B19C0B54-001/shard-4-001/cores}
   [junit4]   2> 252274 ERROR (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 252274 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solrâ„¢ version 8.0.0
   [junit4]   2> 252274 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 252274 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 252274 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-08-29T00:49:33.540Z
   [junit4]   2> 252278 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 252278 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_375EC011B19C0B54-001/shard-4-001/solr.xml
   [junit4]   2> 252283 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@76294c73, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 252287 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:61215/solr
   [junit4]   2> 252300 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [n:127.0.0.1:61302_qup    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 252303 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [n:127.0.0.1:61302_qup    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 252306 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [n:127.0.0.1:61302_qup    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:61302_qup
   [junit4]   2> 252309 INFO  (zkCallback-468-thread-2-processing-n:127.0.0.1:61222_qup) [n:127.0.0.1:61222_qup    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 252309 INFO  (zkCallback-486-thread-1-processing-n:127.0.0.1:61263_qup) [n:127.0.0.1:61263_qup    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 252309 INFO  (zkCallback-475-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 252310 INFO  (zkCallback-480-thread-1-processing-n:127.0.0.1:61246_qup) [n:127.0.0.1:61246_qup    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 252310 INFO  (zkCallback-492-thread-1-processing-n:127.0.0.1:61285_qup) [n:127.0.0.1:61285_qup    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 252310 INFO  (zkCallback-498-thread-1-processing-n:127.0.0.1:61302_qup) [n:127.0.0.1:61302_qup    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 252311 INFO  (OverseerCollectionConfigSetProcessor-98564031268388868-127.0.0.1:61222_qup-n_0000000000) [n:127.0.0.1:61222_qup    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000008 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 252408 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [n:127.0.0.1:61302_qup    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@76294c73
   [junit4]   2> 252416 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [n:127.0.0.1:61302_qup    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@76294c73
   [junit4]   2> 252416 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [n:127.0.0.1:61302_qup    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@76294c73
   [junit4]   2> 252418 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[375EC011B19C0B54]) [n:127.0.0.1:61302_qup    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_375EC011B19C0B54-001/shard-4-001/cores
   [junit4]   2> 252606 INFO  (qtp1533496363-2740) [n:127.0.0.1:61263_qup    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:61302_qup&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 252610 INFO  (OverseerThreadFactory-1483-thread-5-processing-n:127.0.0.1:61222_qup) [n:127.0.0.1:61222_qup    ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:61302_qup for creating new replica
   [junit4]   2> 252614 INFO  (qtp1281687964-2795) [n:127.0.0.1:61302_qup    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n67&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 252615 INFO  (qtp1281687964-2795) [n:127.0.0.1:61302_qup    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 253634 INFO  (qtp1281687964-2795) [n:127.0.0.1:61302_qup c:collection1 s:shard2  x:collection1_shard2_replica_n67] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 253643 INFO  (qtp1281687964-2795) [n:127.0.0.1:61302_qup c:collection1 s:shard2  x:collection1_shard2_replica_n67] o.a.s.s.IndexSchema [collection1_shard2_replica_n67] Schema name=test
   [junit4]   2> 253721 INFO  (qtp1281687964-2795) [n:127.0.0.1:61302_qup c:collection1 s:shard2  x:collection1_shard2_replica_n67] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 253771 INFO  (qtp1281687964-2795) [n:127.0.0.1:61302_qup c:collection1 s:shard2  x:collection1_shard2_replica_n67] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_n67' using configuration from collection collection1, trusted=true
   [junit4]   2> 253771 INFO  (qtp1281687964-2795) [n:127.0.0.1:61302_qup c:collection1 s:shard2  x:collection1_shard2_replica_n67] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_n67' (registry 'solr.core.collection1.shard2.replica_n67') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@76294c73
   [junit4]   2> 253771 INFO  (qtp1281687964-2795) [n:127.0.0.1:61302_qup c:collection1 s:shard2  x:collection1_shard2_replica_n67] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 253771 INFO  (qtp1281687964-2795) [n:127.0.0.1:61302_qup c:collection1 s:shard2  x:collection1_shard2_replica_n67] o.a.s.c.SolrCore [[collection1_shard2_replica_n67] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_375EC011B19C0B54-001/shard-4-001/cores/collection1_shard2_replica_n67], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_375EC011B19C0B54-001/shard-4-001/cores/collection1_shard2_replica_n67/data/]
   [junit4]   2> 253806 INFO  (qtp1281687964-2795) [n:127.0.0.1:61302_qup c:collection1 s:shard2  x:collection1_shard2_replica_n67] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=31, maxMergeAtOnceExplicit=14, maxMergedSegmentMB=65.3056640625, floorSegmentMB=1.2509765625, forceMergeDeletesPctAllowed=11.64130481790744, segmentsPerTier=17.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 253910 WARN  (qtp1281687964-2795) [n:127.0.0.1:61302_qup c:collection1 s:shard2  x:collection1_shard2_replica_n67] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 254047 INFO  (qtp1281687964-2795) [n:127.0.0.1:61302_qup c:collection1 s:shard2  x:collection1_shard2_replica_n67] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 254047 INFO  (qtp1281687964-2795) [n:127.0.0.1:61302_qup c:collection1 s:shard2  x:collection1_shard2_replica_n67] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 254052 INFO  (qtp1281687964-2795) [n:127.0.0.1:61302_qup c:collection1 s:shard2  x:collection1_shard2_replica_n67] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 254052 INFO  (qtp1281687964-2795) [n:127.0.0.1:61302_qup c:collection1 s:shard2  x:collection1_shard2_replica_n67] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 254055 INFO  (qtp1281687964-2795) [n:127.0.0.1:61302_qup c:collection1 s:shard2  x:collection1_shard2_replica_n67] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=17, maxMergeAtOnceExplicit=19, maxMergedSegmentMB=83.12890625, floorSegmentMB=0.98828125, forceMergeDeletesPctAllowed=20.30844128165794, segmentsPerTier=20.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 254055 INFO  (qtp1281687964-2795) [n:127.0.0.1:61302_qup c:collection1 s:shard2  x:collection1_shard2_replica_n67] o.a.s.s.SolrIndexSearcher Opening [Searcher@274c7714[collection1_shard2_replica_n67] main]
   [junit4]   2> 254057 INFO  (qtp1281687964-2795) [n:127.0.0.1:61302_qup c:collection1 s:shard2  x:collection1_shard2_replica_n67] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 254058 INFO  (qtp1281687964-2795) [n:127.0.0.1:61302_qup c:collection1 s:shard2  x:collection1_shard2_replica_n67] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 254060 INFO  (qtp1281687964-2795) [n:127.0.0.1:61302_qup c:collection1 s:shard2  x:collection1_shard2_replica_n67] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 254061 INFO  (searcherExecutor-1530-thread-1-processing-n:127.0.0.1:61302_qup x:collection1_shard2_replica_n67 s:shard2 c:collection1) [n:127.0.0.1:61302_qup c:collection1 s:shard2  x:collection1_shard2_replica_n67] o.a.s.c.SolrCore [collection1_shard2_replica_n67] Registered new searcher Searcher@274c7714[collection1_shard2_replica_n67] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 254062 INFO  (qtp1281687964-2795) [n:127.0.0.1:61302_qup c:collection1 s:shard2  x:collection1_shard2_replica_n67] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1577024513981284352
   [junit4]   2> 254071 INFO  (qtp1281687964-2795) [n:127.0.0.1:61302_qup c:collection1 s:shard2  x:collection1_shard2_replica_n67] o.a.s.c.ZkController Core needs to recover:collection1_shard2_replica_n67
   [junit4]   2> 254074 INFO  (updateExecutor-495-thread-1-processing-n:127.0.0.1:61302_qup x:collection1_shard2_replica_n67 s:shard2 c:collection1) [n:127.0.0.1:61302_qup c:collection1 s:shard2 r:core_node68 x:collection1_shard2_replica_n67] o.a.s.u.DefaultSolrCoreState Running recovery
   [junit4]   2> 254074 INFO  (recoveryExecutor-496-thread-1-processin

[...truncated too long message...]

int] The import com.codahale.metrics.Histogram is never used
 [ecj-lint] ----------
 [ecj-lint] 13. ERROR in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/metrics/reporters/SolrSlf4jReporter.java (at line 27)
 [ecj-lint] 	import com.codahale.metrics.Meter;
 [ecj-lint] 	       ^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] The import com.codahale.metrics.Meter is never used
 [ecj-lint] ----------
 [ecj-lint] 14. ERROR in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/metrics/reporters/SolrSlf4jReporter.java (at line 29)
 [ecj-lint] 	import com.codahale.metrics.MetricRegistry;
 [ecj-lint] 	       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] The import com.codahale.metrics.MetricRegistry is never used
 [ecj-lint] ----------
 [ecj-lint] 15. ERROR in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/metrics/reporters/SolrSlf4jReporter.java (at line 32)
 [ecj-lint] 	import com.codahale.metrics.Timer;
 [ecj-lint] 	       ^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] The import com.codahale.metrics.Timer is never used
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 16. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/request/SimpleFacets.java (at line 925)
 [ecj-lint] 	fastForRandomSet = new HashDocSet(sset.getDocs(), 0, sset.size());
 [ecj-lint] 	^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: 'fastForRandomSet' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 17. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/response/SmileResponseWriter.java (at line 33)
 [ecj-lint] 	new SmileWriter(out, request, response).writeResponse();
 [ecj-lint] 	^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: '<unassigned Closeable value>' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 18. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/schema/OpenExchangeRatesOrgProvider.java (at line 146)
 [ecj-lint] 	ratesJsonStream = resourceLoader.openResource(ratesFileLocation);
 [ecj-lint] 	^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: 'ratesJsonStream' is not closed at this location
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 19. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java (at line 1315)
 [ecj-lint] 	UninvertingReader uninvertingReader = new UninvertingReader(new ReaderWrapper(searcher.getSlowAtomicReader(), collapseField), mapping);
 [ecj-lint] 	                  ^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: 'uninvertingReader' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 20. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/search/DocSlice.java (at line 145)
 [ecj-lint] 	HashDocSet h = new HashDocSet(docs,offset,len);
 [ecj-lint] 	           ^
 [ecj-lint] Resource leak: 'h' is never closed
 [ecj-lint] ----------
 [ecj-lint] 21. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/search/DocSlice.java (at line 154)
 [ecj-lint] 	HashDocSet h = new HashDocSet(docs,offset,len);
 [ecj-lint] 	           ^
 [ecj-lint] Resource leak: 'h' is never closed
 [ecj-lint] ----------
 [ecj-lint] 22. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/search/DocSlice.java (at line 163)
 [ecj-lint] 	HashDocSet h = new HashDocSet(docs,offset,len);
 [ecj-lint] 	           ^
 [ecj-lint] Resource leak: 'h' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 23. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/search/function/OrdFieldSource.java (at line 79)
 [ecj-lint] 	SolrIndexSearcher is = (SolrIndexSearcher) o;
 [ecj-lint] 	                  ^^
 [ecj-lint] Resource leak: 'is' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 24. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/search/function/ReverseOrdFieldSource.java (at line 79)
 [ecj-lint] 	SolrIndexSearcher is = (SolrIndexSearcher) o;
 [ecj-lint] 	                  ^^
 [ecj-lint] Resource leak: 'is' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 25. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java (at line 291)
 [ecj-lint] 	throw new SolrException(ErrorCode.SERVER_ERROR, "Error occurred while loading solr.xml from zookeeper", e);
 [ecj-lint] 	^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 [ecj-lint] Resource leak: 'loader' is not closed at this location
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 26. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/update/CdcrTransactionLog.java (at line 98)
 [ecj-lint] 	ChannelFastInputStream is = new ChannelFastInputStream(channel, pos);
 [ecj-lint] 	                       ^^
 [ecj-lint] Resource leak: 'is' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 27. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/update/HdfsTransactionLog.java (at line 191)
 [ecj-lint] 	LogCodec codec = new LogCodec(resolver);
 [ecj-lint] 	         ^^^^^
 [ecj-lint] Resource leak: 'codec' is never closed
 [ecj-lint] ----------
 [ecj-lint] 28. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/update/HdfsTransactionLog.java (at line 261)
 [ecj-lint] 	LogCodec codec = new LogCodec(resolver);
 [ecj-lint] 	         ^^^^^
 [ecj-lint] Resource leak: 'codec' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 29. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/update/TransactionLog.java (at line 228)
 [ecj-lint] 	ChannelFastInputStream is = new ChannelFastInputStream(channel, pos);
 [ecj-lint] 	                       ^^
 [ecj-lint] Resource leak: 'is' is never closed
 [ecj-lint] ----------
 [ecj-lint] 30. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/update/TransactionLog.java (at line 228)
 [ecj-lint] 	ChannelFastInputStream is = new ChannelFastInputStream(channel, pos);
 [ecj-lint] 	                       ^^
 [ecj-lint] Resource leak: 'is' is never closed
 [ecj-lint] ----------
 [ecj-lint] 31. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/update/TransactionLog.java (at line 260)
 [ecj-lint] 	LogCodec codec = new LogCodec(resolver);
 [ecj-lint] 	         ^^^^^
 [ecj-lint] Resource leak: 'codec' is never closed
 [ecj-lint] ----------
 [ecj-lint] 32. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/update/TransactionLog.java (at line 275)
 [ecj-lint] 	LogCodec codec = new LogCodec(resolver);
 [ecj-lint] 	         ^^^^^
 [ecj-lint] Resource leak: 'codec' is never closed
 [ecj-lint] ----------
 [ecj-lint] 33. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/update/TransactionLog.java (at line 529)
 [ecj-lint] 	LogCodec codec = new LogCodec(resolver);
 [ecj-lint] 	         ^^^^^
 [ecj-lint] Resource leak: 'codec' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 34. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/update/processor/UpdateRequestProcessor.java (at line 85)
 [ecj-lint] 	UpdateRequestProcessor p = this;
 [ecj-lint] 	                       ^
 [ecj-lint] Resource leak: 'p' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 35. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/util/SimplePostTool.java (at line 655)
 [ecj-lint] 	BAOS bos =  new BAOS();
 [ecj-lint] 	     ^^^
 [ecj-lint] Resource leak: 'bos' is never closed
 [ecj-lint] ----------
 [ecj-lint] 36. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/util/SimplePostTool.java (at line 655)
 [ecj-lint] 	BAOS bos =  new BAOS();
 [ecj-lint] 	     ^^^
 [ecj-lint] Resource leak: 'bos' is never closed
 [ecj-lint] ----------
 [ecj-lint] ----------
 [ecj-lint] 37. WARNING in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/java/org/apache/solr/util/SolrLogLayout.java (at line 149)
 [ecj-lint] 	SolrQueryRequest req = requestInfo == null ? null : requestInfo.getReq();
 [ecj-lint] 	                 ^^^
 [ecj-lint] Resource leak: 'req' is never closed
 [ecj-lint] ----------
 [ecj-lint] 37 problems (7 errors, 30 warnings)

BUILD FAILED
/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/build.xml:810: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/build.xml:101: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build.xml:689: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/lucene/common-build.xml:2030: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/lucene/common-build.xml:2069: Compile failed; see the compiler error output for details.

Total time: 87 minutes 19 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any

[JENKINS] Lucene-Solr-master-MacOSX (64bit/jdk1.8.0) - Build # 4152 - Still unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-MacOSX/4152/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseConcMarkSweepGC

2 tests failed.
FAILED:  org.apache.solr.metrics.reporters.SolrSlf4jReporterTest.testReporter

Error Message:
count1=6, count2=6 - no 'solr.node' logs in: {numFound=0,start=0,docs=[]}

Stack Trace:
java.lang.AssertionError: count1=6, count2=6 - no 'solr.node' logs in: {numFound=0,start=0,docs=[]}
	at __randomizedtesting.SeedInfo.seed([D561C3510019443A:8A85EE666B15D77F]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.metrics.reporters.SolrSlf4jReporterTest.testReporter(SolrSlf4jReporterTest.java:90)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.update.processor.UpdateRequestProcessorFactoryTest.testUpdateDistribChainSkipping

Error Message:
Tests must be run with INFO level logging otherwise LogUpdateProcessor isn't used and can't be tested.

Stack Trace:
java.lang.AssertionError: Tests must be run with INFO level logging otherwise LogUpdateProcessor isn't used and can't be tested.
	at __randomizedtesting.SeedInfo.seed([D561C3510019443A:A4853D8776F263F6]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.update.processor.UpdateRequestProcessorFactoryTest.testUpdateDistribChainSkipping(UpdateRequestProcessorFactoryTest.java:101)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 11390 lines...]
   [junit4] Suite: org.apache.solr.update.processor.UpdateRequestProcessorFactoryTest
   [junit4]   2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.update.processor.UpdateRequestProcessorFactoryTest_D561C3510019443A-001/init-core-data-001
   [junit4]   2> 796081 WARN  (SUITE-UpdateRequestProcessorFactoryTest-seed#[D561C3510019443A]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=17 numCloses=17
   [junit4]   2> 796801 WARN  (coreLoadExecutor-4065-thread-1) [    x:collection1] o.a.s.c.RequestHandlers no default request handler is registered (either '/select' or 'standard')
   [junit4]   2> 796804 WARN  (coreLoadExecutor-4065-thread-1) [    x:collection1] o.a.s.r.ManagedResourceStorage Cannot write to config directory /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf; switching to use InMemory storage instead.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=UpdateRequestProcessorFactoryTest -Dtests.method=testUpdateDistribChainSkipping -Dtests.seed=D561C3510019443A -Dtests.slow=true -Dtests.locale=cs -Dtests.timezone=Portugal -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] FAILURE 0.01s J1 | UpdateRequestProcessorFactoryTest.testUpdateDistribChainSkipping <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: Tests must be run with INFO level logging otherwise LogUpdateProcessor isn't used and can't be tested.
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([D561C3510019443A:A4853D8776F263F6]:0)
   [junit4]    > 	at org.apache.solr.update.processor.UpdateRequestProcessorFactoryTest.testUpdateDistribChainSkipping(UpdateRequestProcessorFactoryTest.java:101)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: leaving temporary files on disk at: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.update.processor.UpdateRequestProcessorFactoryTest_D561C3510019443A-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=1497, maxMBSortInHeap=6.89733864185241, sim=RandomSimilarity(queryNorm=true): {}, locale=cs, timezone=Portugal
   [junit4]   2> NOTE: Mac OS X 10.11.6 x86_64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=3,threads=1,free=219817912,total=518979584
   [junit4]   2> NOTE: All tests run in this JVM: [CloudMLTQParserTest, TestBackupRepositoryFactory, TestTolerantUpdateProcessorCloud, DistributedExpandComponentTest, ConnectionManagerTest, DocumentBuilderTest, TestLegacyFieldReuse, SpellPossibilityIteratorTest, TestSQLHandlerNonCloud, TestCloudSchemaless, TestDynamicFieldCollectionResource, TestComponentsName, DistributedFacetPivotWhiteBoxTest, TestRangeQuery, ChaosMonkeySafeLeaderTest, TestManagedSynonymGraphFilterFactory, TestQueryWrapperFilter, ExplicitHLLTest, TestManagedResourceStorage, SecurityConfHandlerTest, TestPivotHelperCode, TestSimpleTrackingShardHandler, TestQueryTypes, TestRealTimeGet, TestElisionMultitermQuery, SpellCheckComponentTest, TestNoOpRegenerator, SchemaVersionSpecificBehaviorTest, SSLMigrationTest, TestRandomRequestDistribution, HdfsRecoverLeaseTest, TestRecovery, SignatureUpdateProcessorFactoryTest, AnalyticsQueryTest, StatelessScriptUpdateProcessorFactoryTest, TestSchemaSimilarityResource, SuggestComponentTest, VMParamsZkACLAndCredentialsProvidersTest, LargeFieldTest, TestUseDocValuesAsStored2, TestGeoJSONResponseWriter, TestMacros, ZkStateReaderTest, TermsComponentTest, PreAnalyzedFieldTest, TestLegacyTerms, TestSchemalessBufferedUpdates, TestRequestStatusCollectionAPI, CdcrUpdateLogTest, PeerSyncWithIndexFingerprintCachingTest, DistributedQueryComponentOptimizationTest, TestPayloadCheckQParserPlugin, DocValuesMissingTest, TestCloudJSONFacetJoinDomain, TestConfigSets, BJQParserTest, ReplicaListTransformerTest, TestPostingsSolrHighlighter, TestPKIAuthenticationPlugin, ChaosMonkeyNothingIsSafeWithPullReplicasTest, TestDistributedMissingSort, SolrShardReporterTest, HdfsRestartWhileUpdatingTest, TestBM25SimilarityFactory, MissingSegmentRecoveryTest, ClassificationUpdateProcessorTest, StatsReloadRaceTest, SolrGangliaReporterTest, TestJoin, SolrMetricReporterTest, JavabinLoaderTest, TermVectorComponentTest, DocValuesMultiTest, PKIAuthenticationIntegrationTest, UUIDUpdateProcessorFallbackTest, ConnectionReuseTest, TestMergePolicyConfig, DistributedFacetPivotSmallAdvancedTest, CdcrReplicationDistributedZkTest, DocValuesNotIndexedTest, TestSolr4Spatial, SolrCoreMetricManagerTest, SmileWriterTest, SolrCLIZkUtilsTest, TestValueSourceCache, TestLegacyNumericRangeQueryBuilder, TestBadConfig, BaseCdcrDistributedZkTest, TestFieldCollectionResource, SparseHLLTest, CollectionStateFormat2Test, UpdateParamsTest, TestRandomFaceting, TestFiltering, TestStressLiveNodes, TestBulkSchemaConcurrent, AutoScalingHandlerTest, TestCoreDiscovery, CopyFieldTest, TestExpandComponent, TermVectorComponentDistributedTest, LeaderElectionContextKeyTest, TlogReplayBufferedWhileIndexingTest, TestSolrDeletionPolicy1, OverseerRolesTest, LukeRequestHandlerTest, HdfsCollectionsAPIDistributedZkTest, AliasIntegrationTest, TestFreeTextSuggestions, TestPrepRecovery, TestCloudSearcherWarming, TestImplicitCoreProperties, TestBulkSchemaAPI, OutOfBoxZkACLAndCredentialsProvidersTest, TestSearcherReuse, TestLocalFSCloudBackupRestore, OverseerCollectionConfigSetProcessorTest, TestCustomStream, UpdateRequestProcessorFactoryTest]
   [junit4] Completed [212/732 (1!)] on J1 in 0.79s, 3 tests, 1 failure <<< FAILURES!

[...truncated 1304 lines...]
   [junit4] Suite: org.apache.solr.metrics.reporters.SolrSlf4jReporterTest
   [junit4]   2> 2806183 INFO  (SUITE-SolrSlf4jReporterTest-seed#[D561C3510019443A]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.metrics.reporters.SolrSlf4jReporterTest_D561C3510019443A-001/init-core-data-001
   [junit4]   2> 2806184 WARN  (SUITE-SolrSlf4jReporterTest-seed#[D561C3510019443A]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=44 numCloses=44
   [junit4]   2> 2806184 INFO  (SUITE-SolrSlf4jReporterTest-seed#[D561C3510019443A]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 2806188 INFO  (SUITE-SolrSlf4jReporterTest-seed#[D561C3510019443A]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN) w/ MAC_OS_X supressed clientAuth
   [junit4]   2> 2806192 INFO  (TEST-SolrSlf4jReporterTest.testReporter-seed#[D561C3510019443A]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testReporter
   [junit4]   2> 2812633 INFO  (TEST-SolrSlf4jReporterTest.testReporter-seed#[D561C3510019443A]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testReporter
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=SolrSlf4jReporterTest -Dtests.method=testReporter -Dtests.seed=D561C3510019443A -Dtests.slow=true -Dtests.locale=es-US -Dtests.timezone=America/Whitehorse -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] FAILURE 6.45s J1 | SolrSlf4jReporterTest.testReporter <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: count1=6, count2=6 - no 'solr.node' logs in: {numFound=0,start=0,docs=[]}
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([D561C3510019443A:8A85EE666B15D77F]:0)
   [junit4]    > 	at org.apache.solr.metrics.reporters.SolrSlf4jReporterTest.testReporter(SolrSlf4jReporterTest.java:90)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 2812634 INFO  (SUITE-SolrSlf4jReporterTest-seed#[D561C3510019443A]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.metrics.reporters.SolrSlf4jReporterTest_D561C3510019443A-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=1256, maxMBSortInHeap=5.436702234765585, sim=RandomSimilarity(queryNorm=true): {}, locale=es-US, timezone=America/Whitehorse
   [junit4]   2> NOTE: Mac OS X 10.11.6 x86_64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=3,threads=1,free=329273856,total=518979584
   [junit4]   2> NOTE: All tests run in this JVM: [CloudMLTQParserTest, TestBackupRepositoryFactory, TestTolerantUpdateProcessorCloud, DistributedExpandComponentTest, ConnectionManagerTest, DocumentBuilderTest, TestLegacyFieldReuse, SpellPossibilityIteratorTest, TestSQLHandlerNonCloud, TestCloudSchemaless, TestDynamicFieldCollectionResource, TestComponentsName, DistributedFacetPivotWhiteBoxTest, TestRangeQuery, ChaosMonkeySafeLeaderTest, TestManagedSynonymGraphFilterFactory, TestQueryWrapperFilter, ExplicitHLLTest, TestManagedResourceStorage, SecurityConfHandlerTest, TestPivotHelperCode, TestSimpleTrackingShardHandler, TestQueryTypes, TestRealTimeGet, TestElisionMultitermQuery, SpellCheckComponentTest, TestNoOpRegenerator, SchemaVersionSpecificBehaviorTest, SSLMigrationTest, TestRandomRequestDistribution, HdfsRecoverLeaseTest, TestRecovery, SignatureUpdateProcessorFactoryTest, AnalyticsQueryTest, StatelessScriptUpdateProcessorFactoryTest, TestSchemaSimilarityResource, SuggestComponentTest, VMParamsZkACLAndCredentialsProvidersTest, LargeFieldTest, TestUseDocValuesAsStored2, TestGeoJSONResponseWriter, TestMacros, ZkStateReaderTest, TermsComponentTest, PreAnalyzedFieldTest, TestLegacyTerms, TestSchemalessBufferedUpdates, TestRequestStatusCollectionAPI, CdcrUpdateLogTest, PeerSyncWithIndexFingerprintCachingTest, DistributedQueryComponentOptimizationTest, TestPayloadCheckQParserPlugin, DocValuesMissingTest, TestCloudJSONFacetJoinDomain, TestConfigSets, BJQParserTest, ReplicaListTransformerTest, TestPostingsSolrHighlighter, TestPKIAuthenticationPlugin, ChaosMonkeyNothingIsSafeWithPullReplicasTest, TestDistributedMissingSort, SolrShardReporterTest, HdfsRestartWhileUpdatingTest, TestBM25SimilarityFactory, MissingSegmentRecoveryTest, ClassificationUpdateProcessorTest, StatsReloadRaceTest, SolrGangliaReporterTest, TestJoin, SolrMetricReporterTest, JavabinLoaderTest, TermVectorComponentTest, DocValuesMultiTest, PKIAuthenticationIntegrationTest, UUIDUpdateProcessorFallbackTest, ConnectionReuseTest, TestMergePolicyConfig, DistributedFacetPivotSmallAdvancedTest, CdcrReplicationDistributedZkTest, DocValuesNotIndexedTest, TestSolr4Spatial, SolrCoreMetricManagerTest, SmileWriterTest, SolrCLIZkUtilsTest, TestValueSourceCache, TestLegacyNumericRangeQueryBuilder, TestBadConfig, BaseCdcrDistributedZkTest, TestFieldCollectionResource, SparseHLLTest, CollectionStateFormat2Test, UpdateParamsTest, TestRandomFaceting, TestFiltering, TestStressLiveNodes, TestBulkSchemaConcurrent, AutoScalingHandlerTest, TestCoreDiscovery, CopyFieldTest, TestExpandComponent, TermVectorComponentDistributedTest, LeaderElectionContextKeyTest, TlogReplayBufferedWhileIndexingTest, TestSolrDeletionPolicy1, OverseerRolesTest, LukeRequestHandlerTest, HdfsCollectionsAPIDistributedZkTest, AliasIntegrationTest, TestFreeTextSuggestions, TestPrepRecovery, TestCloudSearcherWarming, TestImplicitCoreProperties, TestBulkSchemaAPI, OutOfBoxZkACLAndCredentialsProvidersTest, TestSearcherReuse, TestLocalFSCloudBackupRestore, OverseerCollectionConfigSetProcessorTest, TestCustomStream, UpdateRequestProcessorFactoryTest, TestSolrQueryResponse, TestManagedStopFilterFactory, TestMaxTokenLenTokenizer, SolrCoreCheckLockOnStartupTest, ZkSolrClientTest, HdfsBasicDistributedZk2Test, CollectionsAPIDistributedZkTest, CdcrRequestHandlerTest, ShardSplitTest, ZkNodePropsTest, TestSubQueryTransformer, FileUtilsTest, LoggingHandlerTest, ExternalFileFieldSortTest, CustomCollectionTest, TestSystemIdResolver, TestCollationField, TestFastWriter, TestQueryUtils, LeaderFailureAfterFreshStartTest, TestHalfAndHalfDocValues, PolyFieldTest, FacetPivotSmallTest, TestScoreJoinQPScore, ConcurrentDeleteAndCreateCollectionTest, TestJavabinTupleStreamParser, FileBasedSpellCheckerTest, SuggesterFSTTest, TestLuceneMatchVersion, TestLegacyFieldCache, DistribDocExpirationUpdateProcessorTest, DistributedQueryElevationComponentTest, TestNamedUpdateProcessors, DirectSolrSpellCheckerTest, TestConfigReload, TestStressLucene, MoreLikeThisHandlerTest, TestReplicaProperties, DistributedTermsComponentTest, SearchHandlerTest, TestSmileRequest, TestTestInjection, TestSchemaNameResource, ParsingFieldUpdateProcessorsTest, TestSolrQueryParser, TestConfigSetsAPIZkFailure, RulesTest, TestOrdValues, TestSolrXml, CacheHeaderTest, BasicAuthIntegrationTest, BooleanFieldTest, HLLSerializationTest, MoveReplicaHDFSUlogDirTest, DistributedIntervalFacetingTest, SpellCheckCollatorTest, SolrPluginUtilsTest, BasicDistributedZk2Test, OpenCloseCoreStressTest, LeaderElectionIntegrationTest, BasicZkTest, RecoveryZkTest, FullSolrCloudDistribCmdsTest, TestReplicationHandler, ClusterStateUpdateTest, LeaderElectionTest, TestZkChroot, ShardRoutingCustomTest, TestDistributedSearch, AutoCommitTest, TestDistributedGrouping, TestFaceting, TestHashPartitioner, ZkControllerTest, TestReload, HardAutoCommitTest, TestCoreContainer, StatsComponentTest, SolrCmdDistributorTest, PeerSyncTest, BadIndexSchemaTest, ConvertedLegacyTest, TestSort, BasicFunctionalityTest, TestLazyCores, DirectUpdateHandlerTest, TestIndexSearcher, ShowFileRequestHandlerTest, SimplePostToolTest, AnalysisAfterCoreReloadTest, SolrRequestParserTest, TestFoldingMultitermQuery, DocValuesTest, SuggesterTSTTest, TestTrie, TestCSVLoader, TestUpdate, FieldMutatingUpdateProcessorTest, TestAtomicUpdateErrorCases, TestWordDelimiterFilterFactory, QueryEqualityTest, DefaultValueUpdateProcessorTest, SortByFunctionTest, TestRemoteStreaming, DistanceFunctionTest, SolrInfoBeanTest, XsltUpdateRequestHandlerTest, DebugComponentTest, IndexBasedSpellCheckerTest, TestSurroundQueryParser, TestReversedWildcardFilterFactory, DocumentAnalysisRequestHandlerTest, PrimitiveFieldTypeTest, RequestHandlersTest, TestIndexingPerformance, FieldAnalysisRequestHandlerTest, IndexSchemaRuntimeFieldTest, IndexSchemaTest, ReturnFieldsTest, MBeansHandlerTest, TestAnalyzedSuggestions, JsonLoaderTest, TestPartialUpdateDeduplication, PingRequestHandlerTest, CSVRequestHandlerTest, SearchHandlerTest, TestLFUCache, SOLR749Test, TestQuerySenderListener, AlternateDirectoryTest, TestSolrIndexConfig, TestQuerySenderNoQuery, ResponseLogComponentTest, SolrIndexConfigTest, BadComponentTest, TestStressRecovery, TestSolrDeletionPolicy2, SampleTest, TestDocSet, TestSearchPerf, MinimalSchemaTest, TestConfig, OutputWriterTest, TestFuzzyAnalyzedSuggestions, TestSolrCoreProperties, DirectSolrConnectionTest, TestCharFilters, SynonymTokenizerTest, EchoParamsTest, TestSweetSpotSimilarityFactory, TestPerFieldSimilarity, TestLMDirichletSimilarityFactory, TestLMJelinekMercerSimilarityFactory, ResourceLoaderTest, TestFastOutputStream, OpenExchangeRatesOrgProviderTest, URLClassifyProcessorTest, PluginInfoTest, TestFastLRUCache, DateMathParserTest, TestSuggestSpellingConverter, SpellingQueryConverterTest, DOMUtilTest, TestSolrJ, TestLRUCache, TestUtils, TestDocumentBuilder, SliceStateTest, SystemInfoHandlerTest, UUIDFieldTest, DistributedMLTComponentTest, TestRTGBase, CursorPagingTest, SolrTestCaseJ4Test, TestCrossCoreJoin, TestTolerantSearch, TestEmbeddedSolrServerAdminHandler, TestEmbeddedSolrServerConstructors, TestEmbeddedSolrServerSchemaAPI, TestJettySolrRunner, AssignTest, CdcrReplicationHandlerTest, CloudExitableDirectoryReaderTest, CollectionTooManyReplicasTest, DeleteLastCustomShardedReplicaTest, DeleteNodeTest, DeleteReplicaTest, DistribCursorPagingTest, DistributedVersionInfoTest, LeaderInitiatedRecoveryOnCommitTest, LeaderInitiatedRecoveryOnShardRestartTest, MoveReplicaHDFSTest, MultiThreadedOCPTest, RecoveryAfterSoftCommitTest, RemoteQueryErrorTest, ReplaceNodeTest, SharedFSAutoReplicaFailoverTest, SharedFSAutoReplicaFailoverUtilsTest, SolrCloudExampleTest, TestCloudPivotFacet, TestCollectionAPI, TestConfigSetsAPI, TestLeaderInitiatedRecoveryThread, TestPullReplicaErrorHandling, TestRequestForwarding, TestSSLRandomization, TestSegmentSorting, TestSolrCloudWithKerberosAlt, TestSolrCloudWithSecureImpersonation, TestStressCloudBlindAtomicUpdates, CachingDirectoryFactoryTest, TestConfigSetImmutable, TestInfoStreamLogging, TestInitParams, TestReloadAndDeleteDocs, TestShardHandlerFactory, TestSimpleTextCodec, TestSolrCloudSnapshots, TestSolrCoreSnapshots, TestHdfsBackupRestoreCore, TestReplicationHandlerBackup, TestReqParamsAPI, TestRestoreCore, TestSystemCollAutoCreate, CoreMergeIndexesAdminHandlerTest, SegmentsInfoRequestHandlerTest, TestCoreAdminApis, DistributedFacetPivotLargeTest, ShufflingReplicaListTransformerTest, SpatialHeatmapFacetsTest, SolrSlf4jReporterTest]
   [junit4] Completed [622/732 (2!)] on J1 in 6.49s, 1 test, 1 failure <<< FAILURES!

[...truncated 43741 lines...]

[JENKINS] Lucene-Solr-master-MacOSX (64bit/jdk1.8.0) - Build # 4151 - Failure!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-MacOSX/4151/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseParallelGC

1 tests failed.
FAILED:  org.apache.solr.cloud.CustomCollectionTest.testRouteFieldForImplicitRouter

Error Message:
Collection not found: withShardField

Stack Trace:
org.apache.solr.common.SolrException: Collection not found: withShardField
	at __randomizedtesting.SeedInfo.seed([FA7FF63F9C2BF2D2:AF2F1EAD30D23D22]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.getCollectionNames(CloudSolrClient.java:1139)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:822)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
	at org.apache.solr.client.solrj.request.UpdateRequest.commit(UpdateRequest.java:233)
	at org.apache.solr.cloud.CustomCollectionTest.testRouteFieldForImplicitRouter(CustomCollectionTest.java:141)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 12205 lines...]
   [junit4] Suite: org.apache.solr.cloud.CustomCollectionTest
   [junit4]   2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.CustomCollectionTest_FA7FF63F9C2BF2D2-001/init-core-data-001
   [junit4]   2> 1779946 WARN  (SUITE-CustomCollectionTest-seed#[FA7FF63F9C2BF2D2]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=5 numCloses=5
   [junit4]   2> 1779947 INFO  (SUITE-CustomCollectionTest-seed#[FA7FF63F9C2BF2D2]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 1779951 INFO  (SUITE-CustomCollectionTest-seed#[FA7FF63F9C2BF2D2]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN) w/ MAC_OS_X supressed clientAuth
   [junit4]   2> 1779951 INFO  (SUITE-CustomCollectionTest-seed#[FA7FF63F9C2BF2D2]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 1779952 INFO  (SUITE-CustomCollectionTest-seed#[FA7FF63F9C2BF2D2]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 4 servers in /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.CustomCollectionTest_FA7FF63F9C2BF2D2-001/tempDir-001
   [junit4]   2> 1779953 INFO  (SUITE-CustomCollectionTest-seed#[FA7FF63F9C2BF2D2]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 1779953 INFO  (Thread-3793) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1779953 INFO  (Thread-3793) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 1779956 ERROR (Thread-3793) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 1780056 INFO  (SUITE-CustomCollectionTest-seed#[FA7FF63F9C2BF2D2]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:60435
   [junit4]   2> 1780100 INFO  (jetty-launcher-2501-thread-1) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 1780100 INFO  (jetty-launcher-2501-thread-3) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 1780100 INFO  (jetty-launcher-2501-thread-2) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 1780100 INFO  (jetty-launcher-2501-thread-4) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 1780102 INFO  (jetty-launcher-2501-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1a8e35cf{/solr,null,AVAILABLE}
   [junit4]   2> 1780103 INFO  (jetty-launcher-2501-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@5ffaeb36{HTTP/1.1,[http/1.1]}{127.0.0.1:60437}
   [junit4]   2> 1780103 INFO  (jetty-launcher-2501-thread-1) [    ] o.e.j.s.Server Started @1783673ms
   [junit4]   2> 1780103 INFO  (jetty-launcher-2501-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=60437}
   [junit4]   2> 1780103 ERROR (jetty-launcher-2501-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1780103 INFO  (jetty-launcher-2501-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solrâ„¢ version 8.0.0
   [junit4]   2> 1780103 INFO  (jetty-launcher-2501-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1780103 INFO  (jetty-launcher-2501-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 1780104 INFO  (jetty-launcher-2501-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-08-28T03:46:29.645Z
   [junit4]   2> 1780105 INFO  (jetty-launcher-2501-thread-4) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5c353faa{/solr,null,AVAILABLE}
   [junit4]   2> 1780105 INFO  (jetty-launcher-2501-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6382592b{/solr,null,AVAILABLE}
   [junit4]   2> 1780105 INFO  (jetty-launcher-2501-thread-4) [    ] o.e.j.s.AbstractConnector Started ServerConnector@40a4b28f{HTTP/1.1,[http/1.1]}{127.0.0.1:60439}
   [junit4]   2> 1780105 INFO  (jetty-launcher-2501-thread-4) [    ] o.e.j.s.Server Started @1783676ms
   [junit4]   2> 1780105 INFO  (jetty-launcher-2501-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@2da63d76{HTTP/1.1,[http/1.1]}{127.0.0.1:60440}
   [junit4]   2> 1780105 INFO  (jetty-launcher-2501-thread-4) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=60439}
   [junit4]   2> 1780105 INFO  (jetty-launcher-2501-thread-2) [    ] o.e.j.s.Server Started @1783676ms
   [junit4]   2> 1780106 INFO  (jetty-launcher-2501-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=60440}
   [junit4]   2> 1780106 ERROR (jetty-launcher-2501-thread-4) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1780106 ERROR (jetty-launcher-2501-thread-2) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1780106 INFO  (jetty-launcher-2501-thread-4) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solrâ„¢ version 8.0.0
   [junit4]   2> 1780106 INFO  (jetty-launcher-2501-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solrâ„¢ version 8.0.0
   [junit4]   2> 1780106 INFO  (jetty-launcher-2501-thread-4) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1780106 INFO  (jetty-launcher-2501-thread-3) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1bad2d19{/solr,null,AVAILABLE}
   [junit4]   2> 1780106 INFO  (jetty-launcher-2501-thread-4) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 1780106 INFO  (jetty-launcher-2501-thread-4) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-08-28T03:46:29.648Z
   [junit4]   2> 1780106 INFO  (jetty-launcher-2501-thread-3) [    ] o.e.j.s.AbstractConnector Started ServerConnector@65ee3a51{HTTP/1.1,[http/1.1]}{127.0.0.1:60441}
   [junit4]   2> 1780106 INFO  (jetty-launcher-2501-thread-3) [    ] o.e.j.s.Server Started @1783677ms
   [junit4]   2> 1780106 INFO  (jetty-launcher-2501-thread-3) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=60441}
   [junit4]   2> 1780107 ERROR (jetty-launcher-2501-thread-3) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1780106 INFO  (jetty-launcher-2501-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1780107 INFO  (jetty-launcher-2501-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 1780107 INFO  (jetty-launcher-2501-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-08-28T03:46:29.649Z
   [junit4]   2> 1780107 INFO  (jetty-launcher-2501-thread-3) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solrâ„¢ version 8.0.0
   [junit4]   2> 1780107 INFO  (jetty-launcher-2501-thread-3) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1780107 INFO  (jetty-launcher-2501-thread-3) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 1780107 INFO  (jetty-launcher-2501-thread-3) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-08-28T03:46:29.649Z
   [junit4]   2> 1780110 INFO  (jetty-launcher-2501-thread-4) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1780110 INFO  (jetty-launcher-2501-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1780110 INFO  (jetty-launcher-2501-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1780110 INFO  (jetty-launcher-2501-thread-3) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1780124 INFO  (jetty-launcher-2501-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:60435/solr
   [junit4]   2> 1780126 INFO  (jetty-launcher-2501-thread-3) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:60435/solr
   [junit4]   2> 1780126 INFO  (jetty-launcher-2501-thread-4) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:60435/solr
   [junit4]   2> 1780130 INFO  (jetty-launcher-2501-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:60435/solr
   [junit4]   2> 1780334 INFO  (jetty-launcher-2501-thread-3) [n:127.0.0.1:60441_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1780335 INFO  (jetty-launcher-2501-thread-1) [n:127.0.0.1:60437_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1780335 INFO  (jetty-launcher-2501-thread-3) [n:127.0.0.1:60441_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:60441_solr
   [junit4]   2> 1780337 INFO  (jetty-launcher-2501-thread-1) [n:127.0.0.1:60437_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:60437_solr
   [junit4]   2> 1780337 INFO  (jetty-launcher-2501-thread-3) [n:127.0.0.1:60441_solr    ] o.a.s.c.Overseer Overseer (id=98559065423675401-127.0.0.1:60441_solr-n_0000000000) starting
   [junit4]   2> 1780341 INFO  (zkCallback-2520-thread-1-processing-n:127.0.0.1:60441_solr) [n:127.0.0.1:60441_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1780341 INFO  (zkCallback-2523-thread-1-processing-n:127.0.0.1:60437_solr) [n:127.0.0.1:60437_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1780357 INFO  (jetty-launcher-2501-thread-3) [n:127.0.0.1:60441_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:60441_solr
   [junit4]   2> 1780360 INFO  (zkCallback-2523-thread-1-processing-n:127.0.0.1:60437_solr) [n:127.0.0.1:60437_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1780361 INFO  (zkCallback-2520-thread-1-processing-n:127.0.0.1:60441_solr) [n:127.0.0.1:60441_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 1780395 INFO  (jetty-launcher-2501-thread-2) [n:127.0.0.1:60440_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1780399 INFO  (jetty-launcher-2501-thread-2) [n:127.0.0.1:60440_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1780399 INFO  (jetty-launcher-2501-thread-4) [n:127.0.0.1:60439_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 1780403 INFO  (jetty-launcher-2501-thread-2) [n:127.0.0.1:60440_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:60440_solr
   [junit4]   2> 1780405 INFO  (jetty-launcher-2501-thread-4) [n:127.0.0.1:60439_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 1780406 INFO  (zkCallback-2523-thread-1-processing-n:127.0.0.1:60437_solr) [n:127.0.0.1:60437_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1780407 INFO  (zkCallback-2520-thread-1-processing-n:127.0.0.1:60441_solr) [n:127.0.0.1:60441_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1780410 INFO  (zkCallback-2522-thread-1-processing-n:127.0.0.1:60439_solr) [n:127.0.0.1:60439_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1780413 INFO  (zkCallback-2521-thread-1-processing-n:127.0.0.1:60440_solr) [n:127.0.0.1:60440_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 1780422 INFO  (jetty-launcher-2501-thread-4) [n:127.0.0.1:60439_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:60439_solr
   [junit4]   2> 1780424 INFO  (zkCallback-2523-thread-1-processing-n:127.0.0.1:60437_solr) [n:127.0.0.1:60437_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 1780425 INFO  (zkCallback-2520-thread-1-processing-n:127.0.0.1:60441_solr) [n:127.0.0.1:60441_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 1780425 INFO  (zkCallback-2522-thread-1-processing-n:127.0.0.1:60439_solr) [n:127.0.0.1:60439_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 1780425 INFO  (zkCallback-2521-thread-1-processing-n:127.0.0.1:60440_solr) [n:127.0.0.1:60440_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 1780542 INFO  (jetty-launcher-2501-thread-3) [n:127.0.0.1:60441_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_60441.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@25c4da3a
   [junit4]   2> 1780545 INFO  (jetty-launcher-2501-thread-4) [n:127.0.0.1:60439_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_60439.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@25c4da3a
   [junit4]   2> 1780560 INFO  (jetty-launcher-2501-thread-1) [n:127.0.0.1:60437_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_60437.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@25c4da3a
   [junit4]   2> 1780605 INFO  (jetty-launcher-2501-thread-2) [n:127.0.0.1:60440_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_60440.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@25c4da3a
   [junit4]   2> 1780606 INFO  (jetty-launcher-2501-thread-3) [n:127.0.0.1:60441_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_60441.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@25c4da3a
   [junit4]   2> 1780608 INFO  (jetty-launcher-2501-thread-3) [n:127.0.0.1:60441_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_60441.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@25c4da3a
   [junit4]   2> 1780611 INFO  (jetty-launcher-2501-thread-4) [n:127.0.0.1:60439_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_60439.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@25c4da3a
   [junit4]   2> 1780613 INFO  (jetty-launcher-2501-thread-4) [n:127.0.0.1:60439_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_60439.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@25c4da3a
   [junit4]   2> 1780613 INFO  (jetty-launcher-2501-thread-3) [n:127.0.0.1:60441_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.CustomCollectionTest_FA7FF63F9C2BF2D2-001/tempDir-001/node3/.
   [junit4]   2> 1780614 INFO  (jetty-launcher-2501-thread-4) [n:127.0.0.1:60439_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.CustomCollectionTest_FA7FF63F9C2BF2D2-001/tempDir-001/node4/.
   [junit4]   2> 1780614 INFO  (jetty-launcher-2501-thread-1) [n:127.0.0.1:60437_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_60437.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@25c4da3a
   [junit4]   2> 1780615 INFO  (jetty-launcher-2501-thread-1) [n:127.0.0.1:60437_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_60437.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@25c4da3a
   [junit4]   2> 1780616 INFO  (jetty-launcher-2501-thread-1) [n:127.0.0.1:60437_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.CustomCollectionTest_FA7FF63F9C2BF2D2-001/tempDir-001/node1/.
   [junit4]   2> 1780627 INFO  (jetty-launcher-2501-thread-2) [n:127.0.0.1:60440_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_60440.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@25c4da3a
   [junit4]   2> 1780628 INFO  (jetty-launcher-2501-thread-2) [n:127.0.0.1:60440_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_60440.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@25c4da3a
   [junit4]   2> 1780629 INFO  (jetty-launcher-2501-thread-2) [n:127.0.0.1:60440_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.CustomCollectionTest_FA7FF63F9C2BF2D2-001/tempDir-001/node2/.
   [junit4]   2> 1780770 INFO  (SUITE-CustomCollectionTest-seed#[FA7FF63F9C2BF2D2]-worker) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 1780772 INFO  (SUITE-CustomCollectionTest-seed#[FA7FF63F9C2BF2D2]-worker) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:60435/solr ready
   [junit4]   2> 1780951 INFO  (TEST-CustomCollectionTest.testRouteFieldForHashRouter-seed#[FA7FF63F9C2BF2D2]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testRouteFieldForHashRouter
   [junit4]   2> 1780953 INFO  (TEST-CustomCollectionTest.testRouteFieldForHashRouter-seed#[FA7FF63F9C2BF2D2]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 1780966 INFO  (qtp1005572128-16677) [n:127.0.0.1:60437_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=2&collection.configName=conf&maxShardsPerNode=3&router.field=shard_s&name=routeFieldColl&nrtReplicas=2&action=CREATE&numShards=4&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1780969 INFO  (OverseerThreadFactory-8669-thread-1-processing-n:127.0.0.1:60441_solr) [n:127.0.0.1:60441_solr    ] o.a.s.c.CreateCollectionCmd Create collection routeFieldColl
   [junit4]   2> 1781097 INFO  (OverseerStateUpdate-98559065423675401-127.0.0.1:60441_solr-n_0000000000) [n:127.0.0.1:60441_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"routeFieldColl",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"routeFieldColl_shard1_replica_n1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:60439/solr",
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 1781101 INFO  (OverseerStateUpdate-98559065423675401-127.0.0.1:60441_solr-n_0000000000) [n:127.0.0.1:60441_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"routeFieldColl",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"routeFieldColl_shard1_replica_n2",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:60437/solr",
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 1781104 INFO  (OverseerStateUpdate-98559065423675401-127.0.0.1:60441_solr-n_0000000000) [n:127.0.0.1:60441_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"routeFieldColl",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"routeFieldColl_shard2_replica_n4",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:60441/solr",
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 1781108 INFO  (OverseerStateUpdate-98559065423675401-127.0.0.1:60441_solr-n_0000000000) [n:127.0.0.1:60441_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"routeFieldColl",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"routeFieldColl_shard2_replica_n6",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:60440/solr",
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 1781112 INFO  (OverseerStateUpdate-98559065423675401-127.0.0.1:60441_solr-n_0000000000) [n:127.0.0.1:60441_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"routeFieldColl",
   [junit4]   2>   "shard":"shard3",
   [junit4]   2>   "core":"routeFieldColl_shard3_replica_n8",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:60439/solr",
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 1781115 INFO  (OverseerStateUpdate-98559065423675401-127.0.0.1:60441_solr-n_0000000000) [n:127.0.0.1:60441_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"routeFieldColl",
   [junit4]   2>   "shard":"shard3",
   [junit4]   2>   "core":"routeFieldColl_shard3_replica_n10",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:60437/solr",
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 1781119 INFO  (OverseerStateUpdate-98559065423675401-127.0.0.1:60441_solr-n_0000000000) [n:127.0.0.1:60441_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"routeFieldColl",
   [junit4]   2>   "shard":"shard4",
   [junit4]   2>   "core":"routeFieldColl_shard4_replica_n12",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:60441/solr",
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 1781126 INFO  (OverseerStateUpdate-98559065423675401-127.0.0.1:60441_solr-n_0000000000) [n:127.0.0.1:60441_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"routeFieldColl",
   [junit4]   2>   "shard":"shard4",
   [junit4]   2>   "core":"routeFieldColl_shard4_replica_n14",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:60440/solr",
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 1781337 INFO  (qtp1005572128-16679) [n:127.0.0.1:60437_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node5&collection.configName=conf&newCollection=true&name=routeFieldColl_shard1_replica_n2&action=CREATE&numShards=4&collection=routeFieldColl&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 1781338 INFO  (qtp1005572128-16679) [n:127.0.0.1:60437_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 1781338 INFO  (qtp1005572128-16681) [n:127.0.0.1:60437_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node13&collection.configName=conf&newCollection=true&name=routeFieldColl_shard3_replica_n10&action=CREATE&numShards=4&collection=routeFieldColl&shard=shard3&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 1781347 INFO  (qtp82265712-16695) [n:127.0.0.1:60440_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node16&collection.configName=conf&newCollection=true&name=routeFieldColl_shard4_replica_n14&action=CREATE&numShards=4&collection=routeFieldColl&shard=shard4&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 1781348 INFO  (qtp82265712-16695) [n:127.0.0.1:60440_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 1781348 INFO  (qtp82265712-16696) [n:127.0.0.1:60440_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node9&collection.configName=conf&newCollection=true&name=routeFieldColl_shard2_replica_n6&action=CREATE&numShards=4&collection=routeFieldColl&shard=shard2&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 1781348 INFO  (qtp831482304-16701) [n:127.0.0.1:60441_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node15&collection.configName=conf&newCollection=true&name=routeFieldColl_shard4_replica_n12&action=CREATE&numShards=4&collection=routeFieldColl&shard=shard4&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 1781348 INFO  (qtp1354853348-16687) [n:127.0.0.1:60439_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node11&collection.configName=conf&newCollection=true&name=routeFieldColl_shard3_replica_n8&action=CREATE&numShards=4&collection=routeFieldColl&shard=shard3&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 1781348 INFO  (qtp831482304-16701) [n:127.0.0.1:60441_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 1781348 INFO  (qtp1354853348-16687) [n:127.0.0.1:60439_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 1781349 INFO  (qtp831482304-16703) [n:127.0.0.1:60441_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node7&collection.configName=conf&newCollection=true&name=routeFieldColl_shard2_replica_n4&action=CREATE&numShards=4&collection=routeFieldColl&shard=shard2&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 1781349 INFO  (qtp1354853348-16689) [n:127.0.0.1:60439_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=routeFieldColl_shard1_replica_n1&action=CREATE&numShards=4&collection=routeFieldColl&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 1781481 INFO  (zkCallback-2523-thread-1-processing-n:127.0.0.1:60437_solr) [n:127.0.0.1:60437_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/routeFieldColl/state.json] for collection [routeFieldColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1781481 INFO  (zkCallback-2520-thread-1-processing-n:127.0.0.1:60441_solr) [n:127.0.0.1:60441_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/routeFieldColl/state.json] for collection [routeFieldColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1781481 INFO  (zkCallback-2521-thread-1-processing-n:127.0.0.1:60440_solr) [n:127.0.0.1:60440_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/routeFieldColl/state.json] for collection [routeFieldColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1781482 INFO  (zkCallback-2523-thread-2-processing-n:127.0.0.1:60437_solr) [n:127.0.0.1:60437_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/routeFieldColl/state.json] for collection [routeFieldColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1781481 INFO  (zkCallback-2520-thread-2-processing-n:127.0.0.1:60441_solr) [n:127.0.0.1:60441_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/routeFieldColl/state.json] for collection [routeFieldColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1781482 INFO  (zkCallback-2522-thread-2-processing-n:127.0.0.1:60439_solr) [n:127.0.0.1:60439_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/routeFieldColl/state.json] for collection [routeFieldColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1781481 INFO  (zkCallback-2522-thread-1-processing-n:127.0.0.1:60439_solr) [n:127.0.0.1:60439_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/routeFieldColl/state.json] for collection [routeFieldColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1782388 INFO  (qtp1354853348-16689) [n:127.0.0.1:60439_solr c:routeFieldColl s:shard1 r:core_node3 x:routeFieldColl_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 1782388 INFO  (qtp1005572128-16679) [n:127.0.0.1:60437_solr c:routeFieldColl s:shard1 r:core_node5 x:routeFieldColl_shard1_replica_n2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 1782388 INFO  (qtp831482304-16701) [n:127.0.0.1:60441_solr c:routeFieldColl s:shard4 r:core_node15 x:routeFieldColl_shard4_replica_n12] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 1782411 INFO  (qtp831482304-16703) [n:127.0.0.1:60441_solr c:routeFieldColl s:shard2 r:core_node7 x:routeFieldColl_shard2_replica_n4] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 1782411 INFO  (qtp1354853348-16687) [n:127.0.0.1:60439_solr c:routeFieldColl s:shard3 r:core_node11 x:routeFieldColl_shard3_replica_n8] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 1782414 INFO  (qtp1005572128-16681) [n:127.0.0.1:60437_solr c:routeFieldColl s:shard3 r:core_node13 x:routeFieldColl_shard3_replica_n10] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 1782418 INFO  (qtp82265712-16696) [n:127.0.0.1:60440_solr c:routeFieldColl s:shard2 r:core_node9 x:routeFieldColl_shard2_replica_n6] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 1782418 INFO  (qtp82265712-16695) [n:127.0.0.1:60440_solr c:routeFieldColl s:shard4 r:core_node16 x:routeFieldColl_shard4_replica_n14] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
   [junit4]   2> 1782423 INFO  (qtp1005572128-16679) [n:127.0.0.1:60437_solr c:routeFieldColl s:shard1 r:core_node5 x:routeFieldColl_shard1_replica_n2] o.a.s.s.IndexSchema [routeFieldColl_shard1_replica_n2] Schema name=cloud-dynamic
   [junit4]   2> 1782424 INFO  (qtp1354853348-16689) [n:127.0.0.1:60439_solr c:routeFieldColl s:shard1 r:core_node3 x:routeFieldColl_shard1_replica_n1] o.a.s.s.IndexSchema [routeFieldColl_shard1_replica_n1] Schema name=cloud-dynamic
   [junit4]   2> 1782426 INFO  (qtp831482304-16701) [n:127.0.0.1:60441_solr c:routeFieldColl s:shard4 r:core_node15 x:routeFieldColl_shard4_replica_n12] o.a.s.s.IndexSchema [routeFieldColl_shard4_replica_n12] Schema name=cloud-dynamic
   [junit4]   2> 1782446 INFO  (qtp1005572128-16681) [n:127.0.0.1:60437_solr c:routeFieldColl s:shard3 r:core_node13 x:routeFieldColl_shard3_replica_n10] o.a.s.s.IndexSchema [routeFieldColl_shard3_replica_n10] Schema name=cloud-dynamic
   [junit4]   2> 1782446 INFO  (qtp831482304-16703) [n:127.0.0.1:60441_solr c:routeFieldColl s:shard2 r:core_node7 x:routeFieldColl_shard2_replica_n4] o.a.s.s.IndexSchema [routeFieldColl_shard2_replica_n4] Schema name=cloud-dynamic
   [junit4]   2> 1782460 INFO  (qtp82265712-16696) [n:127.0.0.1:60440_solr c:routeFieldColl s:shard2 r:core_node9 x:routeFieldColl_shard2_replica_n6] o.a.s.s.IndexSchema [routeFieldColl_shard2_replica_n6] Schema name=cloud-dynamic
   [junit4]   2> 1782460 INFO  (qtp82265712-16695) [n:127.0.0.1:60440_solr c:routeFieldColl s:shard4 r:core_node16 x:routeFieldColl_shard4_replica_n14] o.a.s.s.IndexSchema [routeFieldColl_shard4_replica_n14] Schema name=cloud-dynamic
   [junit4]   2> 1782475 INFO  (qtp1005572128-16679) [n:127.0.0.1:60437_solr c:routeFieldColl s:shard1 r:core_node5 x:routeFieldColl_shard1_replica_n2] o.a.s.s.IndexSchema Loaded schema cloud-dynamic/1.6 with uniqueid field id
   [junit4]   2> 1782475 INFO  (qtp1005572128-16679) [n:127.0.0.1:60437_solr c:routeFieldColl s:shard1 r:core_node5 x:routeFieldColl_shard1_replica_n2] o.a.s.c.CoreContainer Creating SolrCore 'routeFieldColl_shard1_replica_n2' using configuration from collection routeFieldColl, trusted=true
   [junit4]   2> 1782476 INFO  (qtp1005572128-16679) [n:127.0.0.1:60437_solr c:routeFieldColl s:shard1 r:core_node5 x:routeFieldColl_shard1_replica_n2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_60437.solr.core.routeFieldColl.shard1.replica_n2' (registry 'solr.core.routeFieldColl.shard1.replica_n2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@25c4da3a
   [junit4]   2> 1782476 INFO  (qtp1005572128-16679) [n:127.0.0.1:60437_solr c:routeFieldColl s:shard1 r:core_node5 x:routeFieldColl_shard1_replica_n2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1782477 INFO  (qtp1005572128-16679) [n:127.0.0.1:60437_solr c:routeFieldColl s:shard1 r:core_node5 x:routeFieldColl_shard1_replica_n2] o.a.s.c.SolrCore [[routeFieldColl_shard1_replica_n2] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.CustomCollectionTest_FA7FF63F9C2BF2D2-001/tempDir-001/node1/routeFieldColl_shard1_replica_n2], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.CustomCollectionTest_FA7FF63F9C2BF2D2-001/tempDir-001/node1/./routeFieldColl_shard1_replica_n2/data/]
   [junit4]   2> 1782482 INFO  (qtp831482304-16703) [n:127.0.0.1:60441_solr c:routeFieldColl s:shard2 r:core_node7 x:routeFieldColl_shard2_replica_n4] o.a.s.s.IndexSchema Loaded schema cloud-dynamic/1.6 with uniqueid field id
   [junit4]   2> 1782482 INFO  (qtp831482304-16703) [n:127.0.0.1:60441_solr c:routeFieldColl s:shard2 r:core_node7 x:routeFieldColl_shard2_replica_n4] o.a.s.c.CoreContainer Creating SolrCore 'routeFieldColl_shard2_replica_n4' using configuration from collection routeFieldColl, trusted=true
   [junit4]   2> 1782482 INFO  (qtp831482304-16703) [n:127.0.0.1:60441_solr c:routeFieldColl s:shard2 r:core_node7 x:routeFieldColl_shard2_replica_n4] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_60441.solr.core.routeFieldColl.shard2.replica_n4' (registry 'solr.core.routeFieldColl.shard2.replica_n4') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@25c4da3a
   [junit4]   2> 1782483 INFO  (qtp831482304-16703) [n:127.0.0.1:60441_solr c:routeFieldColl s:shard2 r:core_node7 x:routeFieldColl_shard2_replica_n4] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1782483 INFO  (qtp831482304-16703) [n:127.0.0.1:60441_solr c:routeFieldColl s:shard2 r:core_node7 x:routeFieldColl_shard2_replica_n4] o.a.s.c.SolrCore [[routeFieldColl_shard2_replica_n4] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.CustomCollectionTest_FA7FF63F9C2BF2D2-001/tempDir-001/node3/routeFieldColl_shard2_replica_n4], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.CustomCollectionTest_FA7FF63F9C2BF2D2-001/tempDir-001/node3/./routeFieldColl_shard2_replica_n4/data/]
   [junit4]   2> 1782487 INFO  (qtp1354853348-16687) [n:127.0.0.1:60439_solr c:routeFieldColl s:shard3 r:core_node11 x:routeFieldColl_shard3_replica_n8] o.a.s.s.IndexSchema [routeFieldColl_shard3_replica_n8] Schema name=cloud-dynamic
   [junit4]   2> 1782509 INFO  (qtp1354853348-16689) [n:127.0.0.1:60439_solr c:routeFieldColl s:shard1 r:core_node3 x:routeFieldColl_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema cloud-dynamic/1.6 with uniqueid field id
   [junit4]   2> 1782509 INFO  (qtp1354853348-16689) [n:127.0.0.1:60439_solr c:routeFieldColl s:shard1 r:core_node3 x:routeFieldColl_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'routeFieldColl_shard1_replica_n1' using configuration from collection routeFieldColl, trusted=true
   [junit4]   2> 1782510 INFO  (qtp1354853348-16689) [n:127.0.0.1:60439_solr c:routeFieldColl s:shard1 r:core_node3 x:routeFieldColl_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_60439.solr.core.routeFieldColl.shard1.replica_n1' (registry 'solr.core.routeFieldColl.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@25c4da3a
   [junit4]   2> 1782510 INFO  (qtp1354853348-16689) [n:127.0.0.1:60439_solr c:routeFieldColl s:shard1 r:core_node3 x:routeFieldColl_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1782510 INFO  (qtp1354853348-16689) [n:127.0.0.1:60439_solr c:routeFieldColl s:shard1 r:core_node3 x:routeFieldColl_shard1_replica_n1] o.a.s.c.SolrCore [[routeFieldColl_shard1_replica_n1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.CustomCollectionTest_FA7FF63F9C2BF2D2-001/tempDir-001/node4/routeFieldColl_shard1_replica_n1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.CustomCollectionTest_FA7FF63F9C2BF2D2-001/tempDir-001/node4/./routeFieldColl_shard1_replica_n1/data/]
   [junit4]   2> 1782511 INFO  (qtp1005572128-16681) [n:127.0.0.1:60437_solr c:routeFieldColl s:shard3 r:core_node13 x:routeFieldColl_shard3_replica_n10] o.a.s.s.IndexSchema Loaded schema cloud-dynamic/1.6 with uniqueid field id
   [junit4]   2> 1782511 INFO  (qtp1005572128-16681) [n:127.0.0.1:60437_solr c:routeFieldColl s:shard3 r:core_node13 x:routeFieldColl_shard3_replica_n10] o.a.s.c.CoreContainer Creating SolrCore 'routeFieldColl_shard3_replica_n10' using configuration from collection routeFieldColl, trusted=true
   [junit4]   2> 1782511 INFO  (qtp1005572128-16681) [n:127.0.0.1:60437_solr c:routeFieldColl s:shard3 r:core_node13 x:routeFieldColl_shard3_replica_n10] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_60437.solr.core.routeFieldColl.shard3.replica_n10' (registry 'solr.core.routeFieldColl.shard3.replica_n10') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@25c4da3a
   [junit4]   2> 1782511 INFO  (qtp1005572128-16681) [n:127.0.0.1:60437_solr c:routeFieldColl s:shard3 r:core_node13 x:routeFieldColl_shard3_replica_n10] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1782511 INFO  (qtp1005572128-16681) [n:127.0.0.1:60437_solr c:routeFieldColl s:shard3 r:core_node13 x:routeFieldColl_shard3_replica_n10] o.a.s.c.SolrCore [[routeFieldColl_shard3_replica_n10] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.CustomCollectionTest_FA7FF63F9C2BF2D2-001/tempDir-001/node1/routeFieldColl_shard3_replica_n10], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.CustomCollectionTest_FA7FF63F9C2BF2D2-001/tempDir-001/node1/./routeFieldColl_shard3_replica_n10/data/]
   [junit4]   2> 1782518 INFO  (qtp831482304-16701) [n:127.0.0.1:60441_solr c:routeFieldColl s:shard4 r:core_node15 x:routeFieldColl_shard4_replica_n12] o.a.s.s.IndexSchema Loaded schema cloud-dynamic/1.6 with uniqueid field id
   [junit4]   2> 1782519 INFO  (qtp831482304-16701) [n:127.0.0.1:60441_solr c:routeFieldColl s:shard4 r:core_node15 x:routeFieldColl_shard4_replica_n12] o.a.s.c.CoreContainer Creating SolrCore 'routeFieldColl_shard4_replica_n12' using configuration from collection routeFieldColl, trusted=true
   [junit4]   2> 1782519 INFO  (qtp831482304-16701) [n:127.0.0.1:60441_solr c:routeFieldColl s:shard4 r:core_node15 x:routeFieldColl_shard4_replica_n12] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_60441.solr.core.routeFieldColl.shard4.replica_n12' (registry 'solr.core.routeFieldColl.shard4.replica_n12') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@25c4da3a
   [junit4]   2> 1782519 INFO  (qtp831482304-16701) [n:127.0.0.1:60441_solr c:routeFieldColl s:shard4 r:core_node15 x:routeFieldColl_shard4_replica_n12] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1782519 INFO  (qtp831482304-16701) [n:127.0.0.1:60441_solr c:routeFieldColl s:shard4 r:core_node15 x:routeFieldColl_shard4_replica_n12] o.a.s.c.SolrCore [[routeFieldColl_shard4_replica_n12] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.CustomCollectionTest_FA7FF63F9C2BF2D2-001/tempDir-001/node3/routeFieldColl_shard4_replica_n12], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.CustomCollectionTest_FA7FF63F9C2BF2D2-001/tempDir-001/node3/./routeFieldColl_shard4_replica_n12/data/]
   [junit4]   2> 1782520 INFO  (qtp82265712-16696) [n:127.0.0.1:60440_solr c:routeFieldColl s:shard2 r:core_node9 x:routeFieldColl_shard2_replica_n6] o.a.s.s.IndexSchema Loaded schema cloud-dynamic/1.6 with uniqueid field id
   [junit4]   2> 1782520 INFO  (qtp82265712-16696) [n:127.0.0.1:60440_solr c:routeFieldColl s:shard2 r:core_node9 x:routeFieldColl_shard2_replica_n6] o.a.s.c.CoreContainer Creating SolrCore 'routeFieldColl_shard2_replica_n6' using configuration from collection routeFieldColl, trusted=true
   [junit4]   2> 1782521 INFO  (qtp82265712-16696) [n:127.0.0.1:60440_solr c:routeFieldColl s:shard2 r:core_node9 x:routeFieldColl_shard2_replica_n6] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_60440.solr.core.routeFieldColl.shard2.replica_n6' (registry 'solr.core.routeFieldColl.shard2.replica_n6') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@25c4da3a
   [junit4]   2> 1782521 INFO  (qtp82265712-16696) [n:127.0.0.1:60440_solr c:routeFieldColl s:shard2 r:core_node9 x:routeFieldColl_shard2_replica_n6] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1782521 INFO  (qtp82265712-16696) [n:127.0.0.1:60440_solr c:routeFieldColl s:shard2 r:core_node9 x:routeFieldColl_shard2_replica_n6] o.a.s.c.SolrCore [[routeFieldColl_shard2_replica_n6] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.CustomCollectionTest_FA7FF63F9C2BF2D2-001/tempDir-001/node2/routeFieldColl_shard2_replica_n6], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.CustomCollectionTest_FA7FF63F9C2BF2D2-001/tempDir-001/node2/./routeFieldColl_shard2_replica_n6/data/]
   [junit4]   2> 1782523 INFO  (qtp82265712-16695) [n:127.0.0.1:60440_solr c:routeFieldColl s:shard4 r:core_node16 x:routeFieldColl_shard4_replica_n14] o.a.s.s.IndexSchema Loaded schema cloud-dynamic/1.6 with uniqueid field id
   [junit4]   2> 1782523 INFO  (qtp82265712-16695) [n:127.0.0.1:60440_solr c:routeFieldColl s:shard4 r:core_node16 x:routeFieldColl_shard4_replica_n14] o.a.s.c.CoreContainer Creating SolrCore 'routeFieldColl_shard4_replica_n14' using configuration from collection routeFieldColl, trusted=true
   [junit4]   2> 1782523 INFO  (qtp82265712-16695) [n:127.0.0.1:60440_solr c:routeFieldColl s:shard4 r:core_node16 x:routeFieldColl_shard4_replica_n14] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_60440.solr.core.routeFieldColl.shard4.replica_n14' (registry 'solr.core.routeFieldColl.shard4.replica_n14') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@25c4da3a
   [junit4]   2> 1782524 INFO  (qtp82265712-16695) [n:127.0.0.1:60440_solr c:routeFieldColl s:shard4 r:core_node16 x:routeFieldColl_shard4_replica_n14] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1782524 INFO  (qtp82265712-16695) [n:127.0.0.1:60440_solr c:routeFieldColl s:shard4 r:core_node16 x:routeFieldColl_shard4_replica_n14] o.a.s.c.SolrCore [[routeFieldColl_shard4_replica_n14] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.CustomCollectionTest_FA7FF63F9C2BF2D2-001/tempDir-001/node2/routeFieldColl_shard4_replica_n14], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.CustomCollectionTest_FA7FF63F9C2BF2D2-001/tempDir-001/node2/./routeFieldColl_shard4_replica_n14/data/]
   [junit4]   2> 1782527 INFO  (qtp1354853348-16687) [n:127.0.0.1:60439_solr c:routeFieldColl s:shard3 r:core_node11 x:routeFieldColl_shard3_replica_n8] o.a.s.s.IndexSchema Loaded schema cloud-dynamic/1.6 with uniqueid field id
   [junit4]   2> 1782527 INFO  (qtp1354853348-16687) [n:127.0.0.1:60439_solr c:routeFieldColl s:shard3 r:core_node11 x:routeFieldColl_shard3_replica_n8] o.a.s.c.CoreContainer Creating SolrCore 'routeFieldColl_shard3_replica_n8' using configuration from collection routeFieldColl, trusted=true
   [junit4]   2> 1782527 INFO  (qtp1354853348-16687) [n:127.0.0.1:60439_solr c:routeFieldColl s:shard3 r:core_node11 x:routeFieldColl_shard3_replica_n8] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_60439.solr.core.routeFieldColl.shard3.replica_n8' (registry 'solr.core.routeFieldColl.shard3.replica_n8') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@25c4da3a
   [junit4]   2> 1782527 INFO  (qtp1354853348-16687) [n:127.0.0.1:60439_solr c:routeFieldColl s:shard3 r:core_node11 x:routeFieldColl_shard3_replica_n8] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 1782527 INFO  (qtp1354853348-16687) [n:127.0.0.1:60439_solr c:routeFieldColl s:shard3 r:core_node11 x:routeFieldColl_shard3_replica_n8] o.a.s.c.SolrCore [[routeFieldColl_shard3_replica_n8] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.CustomCollectionTest_FA7FF63F9C2BF2D2-001/tempDir-001/node4/routeFieldColl_shard3_replica_n8], dataDir=[/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.CustomCollectionTest_FA7FF63F9C2BF2D2-001/tempDir-001/node4/./routeFieldColl_shard3_replica_n8/data/]
   [junit4]   2> 1782696 INFO  (qtp1354853348-16689) [n:127.0.0.1:60439_solr c:routeFieldColl s:shard1 r:core_node3 x:routeFieldColl_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1782696 INFO  (qtp1354853348-16689) [n:127.0.0.1:60439_solr c:routeFieldColl s:shard1 r:core_node3 x:routeFieldColl_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1782701 INFO  (qtp1354853348-16689) [n:127.0.0.1:60439_solr c:routeFieldColl s:shard1 r:core_node3 x:routeFieldColl_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1782701 INFO  (qtp1354853348-16689) [n:127.0.0.1:60439_solr c:routeFieldColl s:shard1 r:core_node3 x:routeFieldColl_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1782709 INFO  (qtp1354853348-16689) [n:127.0.0.1:60439_solr c:routeFieldColl s:shard1 r:core_node3 x:routeFieldColl_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6d7cd86c[routeFieldColl_shard1_replica_n1] main]
   [junit4]   2> 1782711 INFO  (qtp82265712-16695) [n:127.0.0.1:60440_solr c:routeFieldColl s:shard4 r:core_node16 x:routeFieldColl_shard4_replica_n14] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1782712 INFO  (qtp82265712-16695) [n:127.0.0.1:60440_solr c:routeFieldColl s:shard4 r:core_node16 x:routeFieldColl_shard4_replica_n14] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1782713 INFO  (qtp82265712-16695) [n:127.0.0.1:60440_solr c:routeFieldColl s:shard4 r:core_node16 x:routeFieldColl_shard4_replica_n14] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1782713 INFO  (qtp82265712-16695) [n:127.0.0.1:60440_solr c:routeFieldColl s:shard4 r:core_node16 x:routeFieldColl_shard4_replica_n14] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1782716 INFO  (qtp1354853348-16689) [n:127.0.0.1:60439_solr c:routeFieldColl s:shard1 r:core_node3 x:routeFieldColl_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 1782717 INFO  (qtp1005572128-16679) [n:127.0.0.1:60437_solr c:routeFieldColl s:shard1 r:core_node5 x:routeFieldColl_shard1_replica_n2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1782717 INFO  (qtp1005572128-16679) [n:127.0.0.1:60437_solr c:routeFieldColl s:shard1 r:core_node5 x:routeFieldColl_shard1_replica_n2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1782717 INFO  (qtp1354853348-16689) [n:127.0.0.1:60439_solr c:routeFieldColl s:shard1 r:core_node3 x:routeFieldColl_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 1782718 INFO  (qtp1354853348-16689) [n:127.0.0.1:60439_solr c:routeFieldColl s:shard1 r:core_node3 x:routeFieldColl_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1782718 INFO  (searcherExecutor-8680-thread-1-processing-n:127.0.0.1:60439_solr x:routeFieldColl_shard1_replica_n1 s:shard1 c:routeFieldColl r:core_node3) [n:127.0.0.1:60439_solr c:routeFieldColl s:shard1 r:core_node3 x:routeFieldColl_shard1_replica_n1] o.a.s.c.SolrCore [routeFieldColl_shard1_replica_n1] Registered new searcher Searcher@6d7cd86c[routeFieldColl_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1782718 INFO  (qtp1354853348-16689) [n:127.0.0.1:60439_solr c:routeFieldColl s:shard1 r:core_node3 x:routeFieldColl_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1576945049676021760
   [junit4]   2> 1782718 INFO  (qtp1005572128-16679) [n:127.0.0.1:60437_solr c:routeFieldColl s:shard1 r:core_node5 x:routeFieldColl_shard1_replica_n2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1782718 INFO  (qtp1005572128-16679) [n:127.0.0.1:60437_solr c:routeFieldColl s:shard1 r:core_node5 x:routeFieldColl_shard1_replica_n2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1782720 INFO  (qtp831482304-16701) [n:127.0.0.1:60441_solr c:routeFieldColl s:shard4 r:core_node15 x:routeFieldColl_shard4_replica_n12] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1782720 INFO  (qtp831482304-16701) [n:127.0.0.1:60441_solr c:routeFieldColl s:shard4 r:core_node15 x:routeFieldColl_shard4_replica_n12] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1782722 INFO  (qtp831482304-16703) [n:127.0.0.1:60441_solr c:routeFieldColl s:shard2 r:core_node7 x:routeFieldColl_shard2_replica_n4] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1782722 INFO  (qtp831482304-16703) [n:127.0.0.1:60441_solr c:routeFieldColl s:shard2 r:core_node7 x:routeFieldColl_shard2_replica_n4] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1782723 INFO  (qtp831482304-16701) [n:127.0.0.1:60441_solr c:routeFieldColl s:shard4 r:core_node15 x:routeFieldColl_shard4_replica_n12] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1782723 INFO  (qtp831482304-16701) [n:127.0.0.1:60441_solr c:routeFieldColl s:shard4 r:core_node15 x:routeFieldColl_shard4_replica_n12] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1782725 INFO  (qtp831482304-16703) [n:127.0.0.1:60441_solr c:routeFieldColl s:shard2 r:core_node7 x:routeFieldColl_shard2_replica_n4] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1782725 INFO  (qtp831482304-16703) [n:127.0.0.1:60441_solr c:routeFieldColl s:shard2 r:core_node7 x:routeFieldColl_shard2_replica_n4] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1782734 INFO  (qtp82265712-16696) [n:127.0.0.1:60440_solr c:routeFieldColl s:shard2 r:core_node9 x:routeFieldColl_shard2_replica_n6] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1782734 INFO  (qtp82265712-16696) [n:127.0.0.1:60440_solr c:routeFieldColl s:shard2 r:core_node9 x:routeFieldColl_shard2_replica_n6] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1782736 INFO  (qtp82265712-16696) [n:127.0.0.1:60440_solr c:routeFieldColl s:shard2 r:core_node9 x:routeFieldColl_shard2_replica_n6] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1782736 INFO  (qtp82265712-16696) [n:127.0.0.1:60440_solr c:routeFieldColl s:shard2 r:core_node9 x:routeFieldColl_shard2_replica_n6] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1782737 INFO  (qtp82265712-16695) [n:127.0.0.1:60440_solr c:routeFieldColl s:shard4 r:core_node16 x:routeFieldColl_shard4_replica_n14] o.a.s.s.SolrIndexSearcher Opening [Searcher@4d20e95c[routeFieldColl_shard4_replica_n14] main]
   [junit4]   2> 1782738 INFO  (qtp1354853348-16687) [n:127.0.0.1:60439_solr c:routeFieldColl s:shard3 r:core_node11 x:routeFieldColl_shard3_replica_n8] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1782739 INFO  (qtp1354853348-16687) [n:127.0.0.1:60439_solr c:routeFieldColl s:shard3 r:core_node11 x:routeFieldColl_shard3_replica_n8] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1782739 INFO  (qtp82265712-16695) [n:127.0.0.1:60440_solr c:routeFieldColl s:shard4 r:core_node16 x:routeFieldColl_shard4_replica_n14] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 1782740 INFO  (qtp82265712-16695) [n:127.0.0.1:60440_solr c:routeFieldColl s:shard4 r:core_node16 x:routeFieldColl_shard4_replica_n14] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 1782741 INFO  (qtp82265712-16695) [n:127.0.0.1:60440_solr c:routeFieldColl s:shard4 r:core_node16 x:routeFieldColl_shard4_replica_n14] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1782741 INFO  (qtp1354853348-16687) [n:127.0.0.1:60439_solr c:routeFieldColl s:shard3 r:core_node11 x:routeFieldColl_shard3_replica_n8] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1782741 INFO  (qtp1354853348-16687) [n:127.0.0.1:60439_solr c:routeFieldColl s:shard3 r:core_node11 x:routeFieldColl_shard3_replica_n8] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1782742 INFO  (searcherExecutor-8684-thread-1-processing-n:127.0.0.1:60440_solr x:routeFieldColl_shard4_replica_n14 s:shard4 c:routeFieldColl r:core_node16) [n:127.0.0.1:60440_solr c:routeFieldColl s:shard4 r:core_node16 x:routeFieldColl_shard4_replica_n14] o.a.s.c.SolrCore [routeFieldColl_shard4_replica_n14] Registered new searcher Searcher@4d20e95c[routeFieldColl_shard4_replica_n14] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1782742 INFO  (qtp82265712-16695) [n:127.0.0.1:60440_solr c:routeFieldColl s:shard4 r:core_node16 x:routeFieldColl_shard4_replica_n14] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1576945049701187584
   [junit4]   2> 1782745 INFO  (qtp1354853348-16689) [n:127.0.0.1:60439_solr c:routeFieldColl s:shard1 r:core_node3 x:routeFieldColl_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for shard shard1: total=2 found=1 timeoutin=9997ms
   [junit4]   2> 1782747 INFO  (qtp1005572128-16679) [n:127.0.0.1:60437_solr c:routeFieldColl s:shard1 r:core_node5 x:routeFieldColl_shard1_replica_n2] o.a.s.s.SolrIndexSearcher Opening [Searcher@bd7fe8[routeFieldColl_shard1_replica_n2] main]
   [junit4]   2> 1782748 INFO  (qtp831482304-16701) [n:127.0.0.1:60441_solr c:routeFieldColl s:shard4 r:core_node15 x:routeFieldColl_shard4_replica_n12] o.a.s.s.SolrIndexSearcher Opening [Searcher@fe83754[routeFieldColl_shard4_replica_n12] main]
   [junit4]   2> 1782748 INFO  (qtp831482304-16703) [n:127.0.0.1:60441_solr c:routeFieldColl s:shard2 r:core_node7 x:routeFieldColl_shard2_replica_n4] o.a.s.s.SolrIndexSearcher Opening [Searcher@4b718a40[routeFieldColl_shard2_replica_n4] main]
   [junit4]   2> 1782749 INFO  (qtp1005572128-16679) [n:127.0.0.1:60437_solr c:routeFieldColl s:shard1 r:core_node5 x:routeFieldColl_shard1_replica_n2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 1782750 INFO  (qtp1005572128-16679) [n:127.0.0.1:60437_solr c:routeFieldColl s:shard1 r:core_node5 x:routeFieldColl_shard1_replica_n2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 1782750 INFO  (qtp831482304-16703) [n:127.0.0.1:60441_solr c:routeFieldColl s:shard2 r:core_node7 x:routeFieldColl_shard2_replica_n4] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 1782751 INFO  (qtp831482304-16703) [n:127.0.0.1:60441_solr c:routeFieldColl s:shard2 r:core_node7 x:routeFieldColl_shard2_replica_n4] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 1782751 INFO  (qtp82265712-16695) [n:127.0.0.1:60440_solr c:routeFieldColl s:shard4 r:core_node16 x:routeFieldColl_shard4_replica_n14] o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for shard shard4: total=2 found=1 timeoutin=9999ms
   [junit4]   2> 1782751 INFO  (qtp831482304-16703) [n:127.0.0.1:60441_solr c:routeFieldColl s:shard2 r:core_node7 x:routeFieldColl_shard2_replica_n4] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1782752 INFO  (qtp1005572128-16679) [n:127.0.0.1:60437_solr c:routeFieldColl s:shard1 r:core_node5 x:routeFieldColl_shard1_replica_n2] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1782752 INFO  (searcherExecutor-8679-thread-1-processing-n:127.0.0.1:60441_solr x:routeFieldColl_shard2_replica_n4 s:shard2 c:routeFieldColl r:core_node7) [n:127.0.0.1:60441_solr c:routeFieldColl s:shard2 r:core_node7 x:routeFieldColl_shard2_replica_n4] o.a.s.c.SolrCore [routeFieldColl_shard2_replica_n4] Registered new searcher Searcher@4b718a40[routeFieldColl_shard2_replica_n4] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1782753 INFO  (qtp831482304-16703) [n:127.0.0.1:60441_solr c:routeFieldColl s:shard2 r:core_node7 x:routeFieldColl_shard2_replica_n4] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1576945049712721920
   [junit4]   2> 1782753 INFO  (searcherExecutor-8678-thread-1-processing-n:127.0.0.1:60437_solr x:routeFieldColl_shard1_replica_n2 s:shard1 c:routeFieldColl r:core_node5) [n:127.0.0.1:60437_solr c:routeFieldColl s:shard1 r:core_node5 x:routeFieldColl_shard1_replica_n2] o.a.s.c.SolrCore [routeFieldColl_shard1_replica_n2] Registered new searcher Searcher@bd7fe8[routeFieldColl_shard1_replica_n2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1782753 INFO  (qtp1005572128-16679) [n:127.0.0.1:60437_solr c:routeFieldColl s:shard1 r:core_node5 x:routeFieldColl_shard1_replica_n2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1576945049712721920
   [junit4]   2> 1782755 INFO  (qtp831482304-16701) [n:127.0.0.1:60441_solr c:routeFieldColl s:shard4 r:core_node15 x:routeFieldColl_shard4_replica_n12] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 1782756 INFO  (qtp831482304-16701) [n:127.0.0.1:60441_solr c:routeFieldColl s:shard4 r:core_node15 x:routeFieldColl_shard4_replica_n12] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 1782757 INFO  (qtp82265712-16696) [n:127.0.0.1:60440_solr c:routeFieldColl s:shard2 r:core_node9 x:routeFieldColl_shard2_replica_n6] o.a.s.s.SolrIndexSearcher Opening [Searcher@eca9f73[routeFieldColl_shard2_replica_n6] main]
   [junit4]   2> 1782758 INFO  (qtp831482304-16701) [n:127.0.0.1:60441_solr c:routeFieldColl s:shard4 r:core_node15 x:routeFieldColl_shard4_replica_n12] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1782758 INFO  (qtp82265712-16696) [n:127.0.0.1:60440_solr c:routeFieldColl s:shard2 r:core_node9 x:routeFieldColl_shard2_replica_n6] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 1782759 INFO  (qtp82265712-16696) [n:127.0.0.1:60440_solr c:routeFieldColl s:shard2 r:core_node9 x:routeFieldColl_shard2_replica_n6] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 1782759 INFO  (searcherExecutor-8682-thread-1-processing-n:127.0.0.1:60441_solr x:routeFieldColl_shard4_replica_n12 s:shard4 c:routeFieldColl r:core_node15) [n:127.0.0.1:60441_solr c:routeFieldColl s:shard4 r:core_node15 x:routeFieldColl_shard4_replica_n12] o.a.s.c.SolrCore [routeFieldColl_shard4_replica_n12] Registered new searcher Searcher@fe83754[routeFieldColl_shard4_replica_n12] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1782759 INFO  (qtp831482304-16701) [n:127.0.0.1:60441_solr c:routeFieldColl s:shard4 r:core_node15 x:routeFieldColl_shard4_replica_n12] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1576945049719013376
   [junit4]   2> 1782760 INFO  (qtp82265712-16696) [n:127.0.0.1:60440_solr c:routeFieldColl s:shard2 r:core_node9 x:routeFieldColl_shard2_replica_n6] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1782760 INFO  (qtp831482304-16703) [n:127.0.0.1:60441_solr c:routeFieldColl s:shard2 r:core_node7 x:routeFieldColl_shard2_replica_n4] o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for shard shard2: total=2 found=1 timeoutin=9999ms
   [junit4]   2> 1782761 INFO  (searcherExecutor-8683-thread-1-processing-n:127.0.0.1:60440_solr x:routeFieldColl_shard2_replica_n6 s:shard2 c:routeFieldColl r:core_node9) [n:127.0.0.1:60440_solr c:routeFieldColl s:shard2 r:core_node9 x:routeFieldColl_shard2_replica_n6] o.a.s.c.SolrCore [routeFieldColl_shard2_replica_n6] Registered new searcher Searcher@eca9f73[routeFieldColl_shard2_replica_n6] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1782761 INFO  (qtp82265712-16696) [n:127.0.0.1:60440_solr c:routeFieldColl s:shard2 r:core_node9 x:routeFieldColl_shard2_replica_n6] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1576945049721110528
   [junit4]   2> 1782762 INFO  (qtp1005572128-16681) [n:127.0.0.1:60437_solr c:routeFieldColl s:shard3 r:core_node13 x:routeFieldColl_shard3_replica_n10] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1782762 INFO  (qtp1005572128-16681) [n:127.0.0.1:60437_solr c:routeFieldColl s:shard3 r:core_node13 x:routeFieldColl_shard3_replica_n10] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 1782765 INFO  (qtp1354853348-16687) [n:127.0.0.1:60439_solr c:routeFieldColl s:shard3 r:core_node11 x:routeFieldColl_shard3_replica_n8] o.a.s.s.SolrIndexSearcher Opening [Searcher@234d4d36[routeFieldColl_shard3_replica_n8] main]
   [junit4]   2> 1782766 INFO  (qtp1354853348-16687) [n:127.0.0.1:60439_solr c:routeFieldColl s:shard3 r:core_node11 x:routeFieldColl_shard3_replica_n8] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 1782766 INFO  (qtp1354853348-16687) [n:127.0.0.1:60439_solr c:routeFieldColl s:shard3 r:core_node11 x:routeFieldColl_shard3_replica_n8] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 1782767 INFO  (qtp1354853348-16687) [n:127.0.0.1:60439_solr c:routeFieldColl s:shard3 r:core_node11 x:routeFieldColl_shard3_replica_n8] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1782767 INFO  (qtp1354853348-16687) [n:127.0.0.1:60439_solr c:routeFieldColl s:shard3 r:core_node11 x:routeFieldColl_shard3_replica_n8] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1576945049727401984
   [junit4]   2> 1782768 INFO  (searcherExecutor-8685-thread-1-processing-n:127.0.0.1:60439_solr x:routeFieldColl_shard3_replica_n8 s:shard3 c:routeFieldColl r:core_node11) [n:127.0.0.1:60439_solr c:routeFieldColl s:shard3 r:core_node11 x:routeFieldColl_shard3_replica_n8] o.a.s.c.SolrCore [routeFieldColl_shard3_replica_n8] Registered new searcher Searcher@234d4d36[routeFieldColl_shard3_replica_n8] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1782769 INFO  (qtp1005572128-16681) [n:127.0.0.1:60437_solr c:routeFieldColl s:shard3 r:core_node13 x:routeFieldColl_shard3_replica_n10] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1782769 INFO  (qtp1005572128-16681) [n:127.0.0.1:60437_solr c:routeFieldColl s:shard3 r:core_node13 x:routeFieldColl_shard3_replica_n10] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1782771 INFO  (qtp1005572128-16681) [n:127.0.0.1:60437_solr c:routeFieldColl s:shard3 r:core_node13 x:routeFieldColl_shard3_replica_n10] o.a.s.s.SolrIndexSearcher Opening [Searcher@5cb31454[routeFieldColl_shard3_replica_n10] main]
   [junit4]   2> 1782772 INFO  (qtp1005572128-16681) [n:127.0.0.1:60437_solr c:routeFieldColl s:shard3 r:core_node13 x:routeFieldColl_shard3_replica_n10] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 1782773 INFO  (qtp1005572128-16681) [n:127.0.0.1:60437_solr c:routeFieldColl s:shard3 r:core_node13 x:routeFieldColl_shard3_replica_n10] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 1782773 INFO  (qtp1354853348-16687) [n:127.0.0.1:60439_solr c:routeFieldColl s:shard3 r:core_node11 x:routeFieldColl_shard3_replica_n8] o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for shard shard3: total=2 found=1 timeoutin=9999ms
   [junit4]   2> 1782774 INFO  (qtp1005572128-16681) [n:127.0.0.1:60437_solr c:routeFieldColl s:shard3 r:core_node13 x:routeFieldColl_shard3_replica_n10] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1782774 INFO  (qtp1005572128-16681) [n:127.0.0.1:60437_solr c:routeFieldColl s:shard3 r:core_node13 x:routeFieldColl_shard3_replica_n10] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1576945049734742016
   [junit4]   2> 1782775 INFO  (searcherExecutor-8681-thread-1-processing-n:127.0.0.1:60437_solr x:routeFieldColl_shard3_replica_n10 s:shard3 c:routeFieldColl r:core_node13) [n:127.0.0.1:60437_solr c:routeFieldColl s:shard3 r:core_node13 x:routeFieldColl_shard3_replica_n10] o.a.s.c.SolrCore [routeFieldColl_shard3_replica_n10] Registered new searcher Searcher@5cb31454[routeFieldColl_shard3_replica_n10] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1782881 INFO  (zkCallback-2522-thread-1-processing-n:127.0.0.1:60439_solr) [n:127.0.0.1:60439_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/routeFieldColl/state.json] for collection [routeFieldColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1782881 INFO  (zkCallback-2523-thread-1-processing-n:127.0.0.1:60437_solr) [n:127.0.0.1:60437_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/routeFieldColl/state.json] for collection [routeFieldColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1782881 INFO  (zkCallback-2520-thread-2-processing-n:127.0.0.1:60441_solr) [n:127.0.0.1:60441_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/routeFieldColl/state.json] for collection [routeFieldColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1782881 INFO  (zkCallback-2520-thread-1-processing-n:127.0.0.1:60441_solr) [n:127.0.0.1:60441_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/routeFieldColl/state.json] for collection [routeFieldColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1782881 INFO  (zkCallback-2521-thread-1-processing-n:127.0.0.1:60440_solr) [n:127.0.0.1:60440_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/routeFieldColl/state.json] for collection [routeFieldColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1782881 INFO  (zkCallback-2522-thread-2-processing-n:127.0.0.1:60439_solr) [n:127.0.0.1:60439_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/routeFieldColl/state.json] for collection [routeFieldColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1782881 INFO  (zkCallback-2523-thread-2-processing-n:127.0.0.1:60437_solr) [n:127.0.0.1:60437_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/routeFieldColl/state.json] for collection [routeFieldColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1783248 INFO  (qtp1354853348-16689) [n:127.0.0.1:60439_solr c:routeFieldColl s:shard1 r:core_node3 x:routeFieldColl_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1783249 INFO  (qtp1354853348-16689) [n:127.0.0.1:60439_solr c:routeFieldColl s:shard1 r:core_node3 x:routeFieldColl_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1783249 INFO  (qtp1354853348-16689) [n:127.0.0.1:60439_solr c:routeFieldColl s:shard1 r:core_node3 x:routeFieldColl_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:60439/solr/routeFieldColl_shard1_replica_n1/
   [junit4]   2> 1783249 INFO  (qtp1354853348-16689) [n:127.0.0.1:60439_solr c:routeFieldColl s:shard1 r:core_node3 x:routeFieldColl_shard1_replica_n1] o.a.s.u.PeerSync PeerSync: core=routeFieldColl_shard1_replica_n1 url=http://127.0.0.1:60439/solr START replicas=[http://127.0.0.1:60437/solr/routeFieldColl_shard1_replica_n2/] nUpdates=100
   [junit4]   2> 1783252 INFO  (qtp1005572128-16682) [n:127.0.0.1:60437_solr c:routeFieldColl s:shard1 r:core_node5 x:routeFieldColl_shard1_replica_n2] o.a.s.c.S.Request [routeFieldColl_shard1_replica_n2]  webapp=/solr path=/get params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1783252 INFO  (qtp82265712-16695) [n:127.0.0.1:60440_solr c:routeFieldColl s:shard4 r:core_node16 x:routeFieldColl_shard4_replica_n14] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1783252 INFO  (qtp82265712-16695) [n:127.0.0.1:60440_solr c:routeFieldColl s:shard4 r:core_node16 x:routeFieldColl_shard4_replica_n14] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1783252 INFO  (qtp82265712-16695) [n:127.0.0.1:60440_solr c:routeFieldColl s:shard4 r:core_node16 x:routeFieldColl_shard4_replica_n14] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:60440/solr/routeFieldColl_shard4_replica_n14/
   [junit4]   2> 1783253 INFO  (qtp82265712-16695) [n:127.0.0.1:60440_solr c:routeFieldColl s:shard4 r:core_node16 x:routeFieldColl_shard4_replica_n14] o.a.s.u.PeerSync PeerSync: core=routeFieldColl_shard4_replica_n14 url=http://127.0.0.1:60440/solr START replicas=[http://127.0.0.1:60441/solr/routeFieldColl_shard4_replica_n12/] nUpdates=100
   [junit4]   2> 1783254 INFO  (qtp831482304-16705) [n:127.0.0.1:60441_solr c:routeFieldColl s:shard4 r:core_node15 x:routeFieldColl_shard4_replica_n12] o.a.s.c.S.Request [routeFieldColl_shard4_replica_n12]  webapp=/solr path=/get params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1783261 INFO  (qtp831482304-16703) [n:127.0.0.1:60441_solr c:routeFieldColl s:shard2 r:core_node7 x:routeFieldColl_shard2_replica_n4] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1783261 INFO  (qtp831482304-16703) [n:127.0.0.1:60441_solr c:routeFieldColl s:shard2 r:core_node7 x:routeFieldColl_shard2_replica_n4] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1783261 INFO  (qtp831482304-16703) [n:127.0.0.1:60441_solr c:routeFieldColl s:shard2 r:core_node7 x:routeFieldColl_shard2_replica_n4] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:60441/solr/routeFieldColl_shard2_replica_n4/
   [junit4]   2> 1783262 INFO  (qtp831482304-16703) [n:127.0.0.1:60441_solr c:routeFieldColl s:shard2 r:core_node7 x:routeFieldColl_shard2_replica_n4] o.a.s.u.PeerSync PeerSync: core=routeFieldColl_shard2_replica_n4 url=http://127.0.0.1:60441/solr START replicas=[http://127.0.0.1:60440/solr/routeFieldColl_shard2_replica_n6/] nUpdates=100
   [junit4]   2> 1783264 INFO  (qtp82265712-16698) [n:127.0.0.1:60440_solr c:routeFieldColl s:shard2 r:core_node9 x:routeFieldColl_shard2_replica_n6] o.a.s.c.S.Request [routeFieldColl_shard2_replica_n6]  webapp=/solr path=/get params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1783274 INFO  (qtp1354853348-16687) [n:127.0.0.1:60439_solr c:routeFieldColl s:shard3 r:core_node11 x:routeFieldColl_shard3_replica_n8] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1783274 INFO  (qtp1354853348-16687) [n:127.0.0.1:60439_solr c:routeFieldColl s:shard3 r:core_node11 x:routeFieldColl_shard3_replica_n8] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1783274 INFO  (qtp1354853348-16687) [n:127.0.0.1:60439_solr c:routeFieldColl s:shard3 r:core_node11 x:routeFieldColl_shard3_replica_n8] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:60439/solr/routeFieldColl_shard3_replica_n8/
   [junit4]   2> 1783274 INFO  (qtp1354853348-16687) [n:127.0.0.1:60439_solr c:routeFieldColl s:shard3 r:core_node11 x:routeFieldColl_shard3_replica_n8] o.a.s.u.PeerSync PeerSync: core=routeFieldColl_shard3_replica_n8 url=http://127.0.0.1:60439/solr START replicas=[http://127.0.0.1:60437/solr/routeFieldColl_shard3_replica_n10/] nUpdates=100
   [junit4]   2> 1783275 INFO  (qtp1005572128-16683) [n:127.0.0.1:60437_solr c:routeFieldColl s:shard3 r:core_node13 x:routeFieldColl_shard3_replica_n10] o.a.s.c.S.Request [routeFieldColl_shard3_replica_n10]  webapp=/solr path=/get params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 1783550 INFO  (qtp1354853348-16689) [n:127.0.0.1:60439_solr c:routeFieldColl s:shard1 r:core_node3 x:routeFieldColl_shard1_replica_n1] o.a.s.u.PeerSync PeerSync: core=routeFieldColl_shard1_replica_n1 url=http://127.0.0.1:60439/solr DONE.  We have no versions.  sync failed.
   [junit4]   2> 1783550 INFO  (qtp1354853348-16689) [n:127.0.0.1:60439_solr c:routeFieldColl s:shard1 r:core_node3 x:routeFieldColl_shard1_replica_n1] o.a.s.c.SyncStrategy Leader's attempt to sync with shard failed, moving to the next candidate
   [junit4]   2> 1783550 INFO  (qtp1354853348-16689) [n:127.0.0.1:60439_solr c:routeFieldColl s:shard1 r:core_node3 x:routeFieldColl_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
   [junit4]   2> 1783550 INFO  (qtp1354853348-16689) [n:127.0.0.1:60439_solr c:routeFieldColl s:shard1 r:core_node3 x:routeFieldColl_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 1783556 INFO  (qtp82265712-16695) [n:127.0.0.1:60440_solr c:routeFieldColl s:shard4 r:core_node16 x:routeFieldColl_shard4_replica_n14] o.a.s.u.PeerSync PeerSync: core=routeFieldColl_shard4_replica_n14 url=http://127.0.0.1:60440/solr DONE.  We have no versions.  sync failed.
   [junit4]   2> 1783556 INFO  (qtp82265712-16695) [n:127.0.0.1:60440_solr c:routeFieldColl s:shard4 r:core_node16 x:routeFieldColl_shard4_replica_n14] o.a.s.c.SyncStrategy Leader's attempt to sync with shard failed, moving to the next candidate
   [junit4]   2> 1783556 INFO  (qtp82265712-16695) [n:127.0.0.1:60440_solr c:routeFieldColl s:shard4 r:core_node16 x:routeFieldColl_shard4_replica_n14] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
   [junit4]   2> 1783556 INFO  (qtp82265712-16695) [n:127.0.0.1:60440_solr c:routeFieldColl s:shard4 r:core_node16 x:routeFieldColl_shard4_replica_n14] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 1783558 INFO  (qtp1354853348-16689) [n:127.0.0.1:60439_solr c:routeFieldColl s:shard1 r:core_node3 x:routeFieldColl_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:60439/solr/routeFieldColl_shard1_replica_n1/ shard1
   [junit4]   2> 1783565 INFO  (qtp82265712-16695) [n:127.0.0.1:60440_solr c:routeFieldColl s:shard4 r:core_node16 x:routeFieldColl_shard4_replica_n14] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:60440/solr/routeFieldColl_shard4_replica_n14/ shard4
   [junit4]   2> 1783569 INFO  (qtp831482304-16703) [n:127.0.0.1:60441_solr c:routeFieldColl s:shard2 r:core_node7 x:routeFieldColl_shard2_replica_n4] o.a.s.u.PeerSync PeerSync: core=routeFieldColl_shard2_replica_n4 url=http://127.0.0.1:60441/solr DONE.  We have no versions.  sync failed.
   [junit4]   2> 1783569 INFO  (qtp831482304-16703) [n:127.0.0.1:60441_solr c:routeFieldColl s:shard2 r:core_node7 x:routeFieldColl_shard2_replica_n4] o.a.s.c.SyncStrategy Leader's attempt to sync with shard failed, moving to the next candidate
   [junit4]   2> 1783569 INFO  (qtp831482304-16703) [n:127.0.0.1:60441_solr c:routeFieldColl s:shard2 r:core_node7 x:routeFieldColl_shard2_replica_n4] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
   [junit4]   2> 1783569 INFO  (qtp831482304-16703) [n:127.0.0.1:60441_solr c:routeFieldColl s:shard2 r:core_node7 x:routeFieldColl_shard2_replica_n4] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 1783578 INFO  (qtp831482304-16703) [n:127.0.0.1:60441_solr c:routeFieldColl s:shard2 r:core_node7 x:routeFieldColl_shard2_replica_n4] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:60441/solr/routeFieldColl_shard2_replica_n4/ shard2
   [junit4]   2> 1783579 INFO  (qtp1354853348-16687) [n:127.0.0.1:60439_solr c:routeFieldColl s:shard3 r:core_node11 x:routeFieldColl_shard3_replica_n8] o.a.s.u.PeerSync PeerSync: core=routeFieldColl_shard3_replica_n8 url=http://127.0.0.1:60439/solr DONE.  We have no versions.  sync failed.
   [junit4]   2> 1783579 INFO  (qtp1354853348-16687) [n:127.0.0.1:60439_solr c:routeFieldColl s:shard3 r:core_node11 x:routeFieldColl_shard3_replica_n8] o.a.s.c.SyncStrategy Leader's attempt to sync with shard failed, moving to the next candidate
   [junit4]   2> 1783579 INFO  (qtp1354853348-16687) [n:127.0.0.1:60439_solr c:routeFieldColl s:shard3 r:core_node11 x:routeFieldColl_shard3_replica_n8] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
   [junit4]   2> 1783579 INFO  (qtp1354853348-16687) [n:127.0.0.1:60439_solr c:routeFieldColl s:shard3 r:core_node11 x:routeFieldColl_shard3_replica_n8] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 1783587 INFO  (qtp1354853348-16687) [n:127.0.0.1:60439_solr c:routeFieldColl s:shard3 r:core_node11 x:routeFieldColl_shard3_replica_n8] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:60439/solr/routeFieldColl_shard3_replica_n8/ shard3
   [junit4]   2> 1783699 INFO  (zkCallback-2523-thread-2-processing-n:127.0.0.1:60437_solr) [n:127.0.0.1:60437_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/routeFieldColl/state.json] for collection [routeFieldColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1783699 INFO  (zkCallback-2522-thread-2-processing-n:127.0.0.1:60439_solr) [n:127.0.0.1:60439_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/routeFieldColl/state.json] for collection [routeFieldColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1783699 INFO  (zkCallback-2520-thread-1-processing-n:127.0.0.1:60441_solr) [n:127.0.0.1:60441_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/routeFieldColl/state.json] for collection [routeFieldColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1783699 INFO  (zkCallback-2520-thread-2-processing-n:127.0.0.1:60441_solr) [n:127.0.0.1:60441_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/routeFieldColl/state.json] for collection [routeFieldColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1783699 INFO  (zkCallback-2523-thread-1-processing-n:127.0.0.1:60437_solr) [n:127.0.0.1:60437_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/routeFieldColl/state.json] for collection [routeFieldColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1783700 INFO  (zkCallback-2521-thread-1-processing-n:127.0.0.1:60440_solr) [n:127.0.0.1:60440_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/routeFieldColl/state.json] for collection [routeFieldColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1783699 INFO  (zkCallback-2522-thread-1-processing-n:127.0.0.1:60439_solr) [n:127.0.0.1:60439_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/routeFieldColl/state.json] for collection [routeFieldColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 1783713 INFO  (qtp1354853348-16689) [n:127.0.0.1:60439_solr c:routeFieldColl s:shard1 r:core_node3 x:routeFieldColl_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 1783715 INFO  (qtp1354853348-16689) [n:127.0.0.1:60439_solr c:routeFieldColl s:shard1 r:core_node3 x:routeFieldColl_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=routeFieldColl_shard1_replica_n1&action=CREATE&numShards=4&collection=routeFieldColl&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=2366
   [junit4]   2> 1783719 INFO  (qtp82265712-16695) [n:127.0.0.1:60440_solr c:routeFieldColl s:shard4 r:core_node16 x:routeFieldColl_shard4_replica_n14] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 1783722 INFO  (qtp82265712-16695) [n:127.0.0.1:60440_solr c:routeFieldColl s:shard4 r:core_node16 x:routeFieldColl_shard4_replica_n1

[...truncated too long message...]

heds-core-api-2.0.0-M15.jar:/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/test-lib/apacheds-core-avl-2.0.0-M15.jar:/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/test-lib/apacheds-core-shared-2.0.0-M15.jar:/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/test-lib/apacheds-i18n-2.0.0-M15.jar:/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/test-lib/apacheds-interceptor-kerberos-2.0.0-M15.jar:/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/test-lib/apacheds-interceptors-admin-2.0.0-M15.jar:/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/test-lib/apacheds-interceptors-authn-2.0.0-M15.jar:/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/test-lib/apacheds-interceptors-authz-2.0.0-M15.jar:/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/test-lib/apacheds-interceptors-changelog-2.0.0-M15.jar:/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/test-lib/apacheds-interceptors-collective-2.0.0-M15.jar:/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/test-lib/apacheds-interceptors-event-2.0.0-M15.jar:/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/test-lib/apacheds-interceptors-exception-2.0.0-M15.jar:/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/test-lib/apacheds-interceptors-journal-2.0.0-M15.jar:/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/test-lib/apacheds-interceptors-normalization-2.0.0-M15.jar:/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/test-lib/apacheds-interceptors-operational-2.0.0-M15.jar:/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/test-lib/apacheds-interceptors-referral-2.0.0-M15.jar:/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/test-lib/apacheds-interceptors-schema-2.0.0-M15.jar:/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/test-lib/apacheds-interceptors-subtree-2.0.0-M15.jar:/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/test-lib/apacheds-interceptors-trigger-2.0.0-M15.jar:/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/test-lib/apacheds-jdbm-partition-2.0.0-M15.jar:/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/test-lib/apacheds-jdbm1-2.0.0-M2.jar:/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/test-lib/apacheds-kerberos-codec-2.0.0-M15.jar:/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/test-lib/apacheds-ldif-partition-2.0.0-M15.jar:/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/test-lib/apacheds-mavibot-partition-2.0.0-M15.jar:/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/test-lib/apacheds-protocol-kerberos-2.0.0-M15.jar:/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/test-lib/apacheds-protocol-ldap-2.0.0-M15.jar:/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/test-lib/apacheds-protocol-shared-2.0.0-M15.jar:/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/test-lib/apacheds-xdbm-partition-2.0.0-M15.jar:/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/test-lib/api-all-1.0.0-M20.jar:/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/test-lib/bcprov-jdk15-1.45.jar:/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/test-lib/byte-buddy-1.6.2.jar:/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/test-lib/commons-collections-3.2.2.jar:/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/test-lib/commons-math3-3.4.1.jar:/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/test-lib/ehcache-core-2.4.4.jar:/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/test-lib/hadoop-common-2.7.4-tests.jar:/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/test-lib/hadoop-hdfs-2.7.4-tests.jar:/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/test-lib/hadoop-minikdc-2.7.4.jar:/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/test-lib/htrace-core-3.2.0-incubating.jar:/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/test-lib/jersey-core-1.9.jar:/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/test-lib/jersey-server-1.9.jar:/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/test-lib/jetty-6.1.26.jar:/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/test-lib/jetty-sslengine-6.1.26.jar:/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/test-lib/jetty-util-6.1.26.jar:/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/test-lib/mina-core-2.0.0-M5.jar:/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/test-lib/mockito-core-2.6.2.jar:/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/test-lib/netty-all-4.0.36.Final.jar:/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/test-lib/objenesis-2.5.jar:/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/lucene/build/analysis/icu/lucene-analyzers-icu-8.0.0-SNAPSHOT.jar:/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/contrib/solr-analysis-extras/classes/java:/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/lucene/build/queryparser/classes/test:/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/contrib/analysis-extras/lib/icu4j-59.1.jar:/Users/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-launcher.jar:/Users/jenkins/.ant/lib/ivy-2.3.0.jar:/Users/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-antlr.jar:/Users/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-apache-bcel.jar:/Users/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-apache-bsf.jar:/Users/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-apache-log4j.jar:/Users/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-apache-oro.jar:/Users/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-apache-regexp.jar:/Users/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-apache-resolver.jar:/Users/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-apache-xalan2.jar:/Users/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-commons-logging.jar:/Users/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-commons-net.jar:/Users/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-jai.jar:/Users/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-javamail.jar:/Users/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-jdepend.jar:/Users/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-jmf.jar:/Users/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-jsch.jar:/Users/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-junit.jar:/Users/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-junit4.jar:/Users/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-netrexx.jar:/Users/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-swing.jar:/Users/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-testutil.jar:/Users/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant.jar:/Library/Java/JavaVirtualMachines/jdk1.8.0_144.jdk/Contents/Home/lib/tools.jar:/Users/jenkins/.ivy2/cache/com.carrotsearch.randomizedtesting/junit4-ant/jars/junit4-ant-2.5.2.jar com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe -eventsfile /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/temp/junit4-J0-20170828_031645_8883159019529487111582.events @/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/temp/junit4-J0-20170828_031645_888947467668366641961.suites -stdin
   [junit4] ERROR: JVM J0 ended with an exception: Forked process returned with error code: 134. Very likely a JVM crash.  See process stdout at: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/temp/junit4-J0-20170828_031645_8887894648822902927012.sysout
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4.executeSlave(JUnit4.java:1519)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4.access$000(JUnit4.java:126)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:982)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:979)
   [junit4] 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4] 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4] 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4] 	at java.lang.Thread.run(Thread.java:748)

BUILD FAILED
/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/build.xml:810: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/build.xml:754: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/build.xml:59: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build.xml:267: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/common-build.xml:549: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/lucene/common-build.xml:1458: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-master-MacOSX/lucene/common-build.xml:985: At least one slave process threw an exception, first: Forked process returned with error code: 134. Very likely a JVM crash.  See process stdout at: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/temp/junit4-J0-20170828_031645_8887894648822902927012.sysout

Total time: 85 minutes 38 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any

[JENKINS] Lucene-Solr-master-MacOSX (64bit/jdk1.8.0) - Build # 4150 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-MacOSX/4150/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseSerialGC

1 tests failed.
FAILED:  org.apache.solr.metrics.reporters.SolrSlf4jReporterTest.testReporter

Error Message:
count1=6, count2=6 - no 'solr.node' logs in: {numFound=0,start=0,docs=[]}

Stack Trace:
java.lang.AssertionError: count1=6, count2=6 - no 'solr.node' logs in: {numFound=0,start=0,docs=[]}
	at __randomizedtesting.SeedInfo.seed([9A9A0D0A5342CB91:C57E203D384E58D4]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.metrics.reporters.SolrSlf4jReporterTest.testReporter(SolrSlf4jReporterTest.java:90)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 12780 lines...]
   [junit4] Suite: org.apache.solr.metrics.reporters.SolrSlf4jReporterTest
   [junit4]   2> 3074086 INFO  (SUITE-SolrSlf4jReporterTest-seed#[9A9A0D0A5342CB91]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.metrics.reporters.SolrSlf4jReporterTest_9A9A0D0A5342CB91-001/init-core-data-001
   [junit4]   2> 3074087 WARN  (SUITE-SolrSlf4jReporterTest-seed#[9A9A0D0A5342CB91]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=6 numCloses=6
   [junit4]   2> 3074090 INFO  (SUITE-SolrSlf4jReporterTest-seed#[9A9A0D0A5342CB91]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 3074095 INFO  (SUITE-SolrSlf4jReporterTest-seed#[9A9A0D0A5342CB91]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN) w/ MAC_OS_X supressed clientAuth
   [junit4]   2> 3074099 INFO  (TEST-SolrSlf4jReporterTest.testReporter-seed#[9A9A0D0A5342CB91]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testReporter
   [junit4]   2> 3080548 INFO  (TEST-SolrSlf4jReporterTest.testReporter-seed#[9A9A0D0A5342CB91]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testReporter
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=SolrSlf4jReporterTest -Dtests.method=testReporter -Dtests.seed=9A9A0D0A5342CB91 -Dtests.slow=true -Dtests.locale=cs-CZ -Dtests.timezone=Brazil/East -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] FAILURE 6.46s J1 | SolrSlf4jReporterTest.testReporter <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: count1=6, count2=6 - no 'solr.node' logs in: {numFound=0,start=0,docs=[]}
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([9A9A0D0A5342CB91:C57E203D384E58D4]:0)
   [junit4]    > 	at org.apache.solr.metrics.reporters.SolrSlf4jReporterTest.testReporter(SolrSlf4jReporterTest.java:90)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 3080552 INFO  (SUITE-SolrSlf4jReporterTest-seed#[9A9A0D0A5342CB91]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.metrics.reporters.SolrSlf4jReporterTest_9A9A0D0A5342CB91-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=1515, maxMBSortInHeap=5.510937628923985, sim=RandomSimilarity(queryNorm=true): {}, locale=cs-CZ, timezone=Brazil/East
   [junit4]   2> NOTE: Mac OS X 10.11.6 x86_64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=3,threads=1,free=149326824,total=518979584
   [junit4]   2> NOTE: All tests run in this JVM: [TestCoreDiscovery, TestWordDelimiterFilterFactory, AtomicUpdateProcessorFactoryTest, TestLRUCache, LargeFieldTest, UpdateRequestProcessorFactoryTest, TestSchemaNameResource, TestSha256AuthenticationProvider, TestManagedSynonymGraphFilterFactory, HdfsThreadLeakTest, SpellCheckComponentTest, DeleteReplicaTest, JavabinLoaderTest, FullSolrCloudDistribCmdsTest, MoveReplicaHDFSUlogDirTest, SchemaVersionSpecificBehaviorTest, TestRandomFaceting, SolrShardReporterTest, DocumentAnalysisRequestHandlerTest, PingRequestHandlerTest, AutoScalingHandlerTest, SpellCheckCollatorWithCollapseTest, TestLegacyNumericRangeQueryBuilder, CurrencyFieldTypeTest, SparseHLLTest, TestOrdValues, TestHashQParserPlugin, ZkNodePropsTest, TestReplicationHandlerBackup, DeleteStatusTest, MetricsHandlerTest, DebugComponentTest, TestBackupRepositoryFactory, TestPointFields, CollectionsAPIAsyncDistributedZkTest, ClassificationUpdateProcessorFactoryTest, StressHdfsTest, ZkStateReaderTest, TestJoin, CoreAdminCreateDiscoverTest, SmileWriterTest, TestPartialUpdateDeduplication, ShardRoutingCustomTest, TestConfigSets, TestSerializedLuceneMatchVersion, TestFastWriter, CdcrUpdateLogTest, ConnectionManagerTest, ReplaceNodeTest, TestSearchPerf, SolrCLIZkUtilsTest, LukeRequestHandlerTest, DocumentBuilderTest, TestRangeQuery, SolrGangliaReporterTest, SOLR749Test, SuggesterFSTTest, SearchHandlerTest, TestRTGBase, ClassificationUpdateProcessorTest, TestLuceneMatchVersion, SecurityConfHandlerTest, HLLSerializationTest, TestPKIAuthenticationPlugin, DistributedQueryComponentOptimizationTest, UninvertDocValuesMergePolicyTest, FacetPivotSmallTest, TestAddFieldRealTimeGet, TestManagedSchemaAPI, DocExpirationUpdateProcessorFactoryTest, ExplicitHLLTest, TestLeaderElectionZkExpiry, PreAnalyzedFieldTest, TestCloudSchemaless, TestSearcherReuse, TestImpersonationWithHadoopAuth, TestConfig, CacheHeaderTest, ShardSplitTest, TestScoreJoinQPScore, TestTrie, TestUnifiedSolrHighlighter, FieldMutatingUpdateProcessorTest, TestDistribIDF, TestRestoreCore, CdcrReplicationDistributedZkTest, TestBM25SimilarityFactory, HealthCheckHandlerTest, TestSchemalessBufferedUpdates, TestReload, TestFieldCache, HdfsRecoverLeaseTest, TestXmlQParserPlugin, FileBasedSpellCheckerTest, TestSegmentSorting, BasicFunctionalityTest, TestRawResponseWriter, TestRecoveryHdfs, HdfsCollectionsAPIDistributedZkTest, GraphQueryTest, TestLMJelinekMercerSimilarityFactory, DistributedFacetPivotLargeTest, TestBlendedInfixSuggestions, LeaderElectionContextKeyTest, DOMUtilTest, LoggingHandlerTest, BasicDistributedZk2Test, TestIndexSearcher, CopyFieldTest, CdcrRequestHandlerTest, AnalyticsQueryTest, TestConfigReload, OpenExchangeRatesOrgProviderTest, TestCollectionAPIs, AssignTest, LeaderInitiatedRecoveryOnShardRestartTest, TestRandomRequestDistribution, TestInPlaceUpdatesStandalone, JvmMetricsTest, TestCrossCoreJoin, TestSolrCoreParser, MissingSegmentRecoveryTest, BasicZkTest, SharedFSAutoReplicaFailoverUtilsTest, TestPayloadCheckQParserPlugin, SignatureUpdateProcessorFactoryTest, TestFastOutputStream, TestSQLHandlerNonCloud, TestSolrJ, BooleanFieldTest, TestSolrCloudWithKerberosAlt, TestFieldCacheSort, TermVectorComponentTest, TestHalfAndHalfDocValues, DistribDocExpirationUpdateProcessorTest, TestCSVLoader, TestSolrQueryParser, DirectSolrSpellCheckerTest, TestStressLiveNodes, TestDownShardTolerantSearch, TestJavabinTupleStreamParser, PeerSyncWithIndexFingerprintCachingTest, CheckHdfsIndexTest, CustomCollectionTest, ExternalFileFieldSortTest, TestCharFilters, DirectoryFactoryTest, TestSolr4Spatial, TestEmbeddedSolrServerAdminHandler, TestEmbeddedSolrServerSchemaAPI, TestStandardQParsers, FileUtilsTest, ShowFileRequestHandlerTest, TestMergePolicyConfig, SpellPossibilityIteratorTest, MultiThreadedOCPTest, LeaderElectionIntegrationTest, DistributedTermsComponentTest, TestPerFieldSimilarity, SearchHandlerTest, TestStressCloudBlindAtomicUpdates, TestMiniSolrCloudClusterSSL, TestFilteredDocIdSet, TestCloudSearcherWarming, TestDistributedMissingSort, V2ApiIntegrationTest, CoreMergeIndexesAdminHandlerTest, TestManagedStopFilterFactory, TestCoreAdminApis, TestFreeTextSuggestions, TestSolrCoreSnapshots, SpatialHeatmapFacetsTest, TestConfigSetImmutable, BigEndianAscendingWordSerializerTest, TestNoOpRegenerator, TestCustomSort, CloudMLTQParserTest, TestDistributedGrouping, ChaosMonkeySafeLeaderTest, CollectionsAPIDistributedZkTest, SyncSliceTest, OpenCloseCoreStressTest, OverseerTest, RecoveryZkTest, ClusterStateUpdateTest, ZkSolrClientTest, TestZkChroot, TestRandomDVFaceting, ZkCLITest, TestDistributedSearch, DistributedSpellCheckComponentTest, ZkControllerTest, HardAutoCommitTest, SolrCmdDistributorTest, QueryElevationComponentTest, PeerSyncTest, DirectUpdateHandlerTest, TestBadConfig, TestExtendedDismaxParser, SpellCheckCollatorTest, CoreAdminHandlerTest, SuggesterTSTTest, TestStressLucene, SpatialFilterTest, SuggesterWFSTTest, PolyFieldTest, SolrCoreCheckLockOnStartupTest, TestPseudoReturnFields, QueryEqualityTest, TestRemoteStreaming, DocValuesMultiTest, TestSolrDeletionPolicy1, DisMaxRequestHandlerTest, TestQueryUtils, TestWriterPerf, TermsComponentTest, SolrPluginUtilsTest, TestCollationField, QueryParsingTest, TestAnalyzedSuggestions, UniqFieldsUpdateProcessorFactoryTest, JsonLoaderTest, TestPHPSerializedResponseWriter, TestComponentsName, TestLFUCache, HighlighterConfigTest, UpdateParamsTest, AlternateDirectoryTest, TestSolrIndexConfig, SolrIndexConfigTest, BadComponentTest, MultiTermTest, SampleTest, TestDocSet, TestBinaryField, MinimalSchemaTest, TestElisionMultitermQuery, OutputWriterTest, TestFuzzyAnalyzedSuggestions, TestPostingsSolrHighlighter, SynonymTokenizerTest, EchoParamsTest, TestSweetSpotSimilarityFactory, TestLMDirichletSimilarityFactory, TimeZoneUtilsTest, PluginInfoTest, TestFastLRUCache, TestSystemIdResolver, TestSuggestSpellingConverter, SpellingQueryConverterTest, ClusterStateTest, TestUtils, SliceStateTest, CursorPagingTest, TestSimpleTrackingShardHandler, AliasIntegrationTest, BaseCdcrDistributedZkTest, ChaosMonkeyNothingIsSafeWithPullReplicasTest, CleanupOldIndexTest, CloudExitableDirectoryReaderTest, CollectionReloadTest, CollectionsAPISolrJTest, DocValuesNotIndexedTest, LeaderFailureAfterFreshStartTest, MigrateRouteKeyTest, OutOfBoxZkACLAndCredentialsProvidersTest, OverseerCollectionConfigSetProcessorTest, OverseerModifyCollectionTest, OverseerRolesTest, OverseerStatusTest, OverseerTaskQueueTest, PeerSyncReplicationTest, SSLMigrationTest, SolrCloudExampleTest, TestCloudDeleteByQuery, TestCloudJSONFacetJoinDomain, TestCloudPseudoReturnFields, TestCloudRecovery, TestCollectionsAPIViaSolrCloudCluster, TestConfigSetsAPIZkFailure, TestCryptoKeys, TestDeleteCollectionOnDownNodes, TestExclusionRuleCollectionAccess, TestHdfsCloudBackupRestore, TestLeaderElectionWithEmptyReplica, TestLeaderInitiatedRecoveryThread, TestOnReconnectListenerSupport, TestPrepRecovery, TestPullReplicaErrorHandling, TestReplicaProperties, TestSizeLimitedDistributedMap, TestTlogReplica, BlobRepositoryCloudTest, ConfigureRecoveryStrategyTest, ExitableDirectoryReaderTest, HdfsDirectoryFactoryTest, TestConfigOverlay, TestConfigSetProperties, TestDynamicLoading, TestSolrXml, TestBlobHandler, TestSystemCollAutoCreate, TestConfigsApi, DistributedExpandComponentTest, DistributedFacetPivotLongTailTest, DistributedFacetPivotSmallTest, ReplicaListTransformerTest, SuggestComponentTest, TestExpandComponent, TestHttpShardHandlerFactory, TestPivotHelperCode, TestTrackingShardHandlerFactory, HighlighterMaxOffsetTest, WrapperMergePolicyFactoryTest, TestLegacyField, TestLegacyFieldReuse, TestLegacyNumericUtils, TestLegacyTerms, TestMultiValuedNumericRangeQuery, TestNumericRangeQuery32, TestNumericTokenStream, SolrCoreMetricManagerTest, SolrMetricManagerTest, SolrMetricReporterTest, SolrMetricsIntegrationTest, SolrJmxReporterTest, SolrSlf4jReporterTest]
   [junit4] Completed [625/732 (1!)] on J1 in 6.50s, 1 test, 1 failure <<< FAILURES!

[...truncated 43675 lines...]

[JENKINS] Lucene-Solr-master-MacOSX (64bit/jdk1.8.0) - Build # 4149 - Still unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-MacOSX/4149/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseParallelGC

2 tests failed.
FAILED:  org.apache.solr.metrics.reporters.SolrSlf4jReporterTest.testReporter

Error Message:
count1=6, count2=6 - no 'solr.node' logs in: {numFound=0,start=0,docs=[]}

Stack Trace:
java.lang.AssertionError: count1=6, count2=6 - no 'solr.node' logs in: {numFound=0,start=0,docs=[]}
	at __randomizedtesting.SeedInfo.seed([AE83759C898BDB1D:F16758ABE2874858]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.metrics.reporters.SolrSlf4jReporterTest.testReporter(SolrSlf4jReporterTest.java:90)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.update.processor.UpdateRequestProcessorFactoryTest.testUpdateDistribChainSkipping

Error Message:
Tests must be run with INFO level logging otherwise LogUpdateProcessor isn't used and can't be tested.

Stack Trace:
java.lang.AssertionError: Tests must be run with INFO level logging otherwise LogUpdateProcessor isn't used and can't be tested.
	at __randomizedtesting.SeedInfo.seed([AE83759C898BDB1D:DF678B4AFF60FCD1]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.update.processor.UpdateRequestProcessorFactoryTest.testUpdateDistribChainSkipping(UpdateRequestProcessorFactoryTest.java:101)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 11158 lines...]
   [junit4] Suite: org.apache.solr.metrics.reporters.SolrSlf4jReporterTest
   [junit4]   2> 679914 INFO  (SUITE-SolrSlf4jReporterTest-seed#[AE83759C898BDB1D]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.metrics.reporters.SolrSlf4jReporterTest_AE83759C898BDB1D-001/init-core-data-001
   [junit4]   2> 679915 WARN  (SUITE-SolrSlf4jReporterTest-seed#[AE83759C898BDB1D]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=146 numCloses=146
   [junit4]   2> 679915 INFO  (SUITE-SolrSlf4jReporterTest-seed#[AE83759C898BDB1D]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 679918 INFO  (SUITE-SolrSlf4jReporterTest-seed#[AE83759C898BDB1D]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN) w/ MAC_OS_X supressed clientAuth
   [junit4]   2> 679919 INFO  (TEST-SolrSlf4jReporterTest.testReporter-seed#[AE83759C898BDB1D]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testReporter
   [junit4]   2> 686265 INFO  (TEST-SolrSlf4jReporterTest.testReporter-seed#[AE83759C898BDB1D]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testReporter
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=SolrSlf4jReporterTest -Dtests.method=testReporter -Dtests.seed=AE83759C898BDB1D -Dtests.slow=true -Dtests.locale=en-NZ -Dtests.timezone=Etc/GMT-0 -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] FAILURE 6.36s J1 | SolrSlf4jReporterTest.testReporter <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: count1=6, count2=6 - no 'solr.node' logs in: {numFound=0,start=0,docs=[]}
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([AE83759C898BDB1D:F16758ABE2874858]:0)
   [junit4]    > 	at org.apache.solr.metrics.reporters.SolrSlf4jReporterTest.testReporter(SolrSlf4jReporterTest.java:90)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 686274 INFO  (SUITE-SolrSlf4jReporterTest-seed#[AE83759C898BDB1D]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.metrics.reporters.SolrSlf4jReporterTest_AE83759C898BDB1D-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=1055, maxMBSortInHeap=6.927893722262716, sim=RandomSimilarity(queryNorm=false): {}, locale=en-NZ, timezone=Etc/GMT-0
   [junit4]   2> NOTE: Mac OS X 10.11.6 x86_64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=3,threads=1,free=61314584,total=367525888
   [junit4]   2> NOTE: All tests run in this JVM: [TestRandomRequestDistribution, BJQParserTest, TestSolrXml, TestFiltering, TestLeaderElectionZkExpiry, JvmMetricsTest, TestSuggestSpellingConverter, TestReloadAndDeleteDocs, ClassificationUpdateProcessorFactoryTest, FileUtilsTest, JavabinLoaderTest, TestSubQueryTransformerDistrib, ExplicitHLLTest, TestConfig, TestClassicSimilarityFactory, TestQueryUtils, TestRecoveryHdfs, DocumentAnalysisRequestHandlerTest, TestIndexSearcher, AutoCommitTest, TlogReplayBufferedWhileIndexingTest, ResponseHeaderTest, BigEndianAscendingWordSerializerTest, DocValuesMissingTest, SSLMigrationTest, DistributedMLTComponentTest, ExternalFileFieldSortTest, AtomicUpdateProcessorFactoryTest, TestInPlaceUpdatesStandalone, TestSolr4Spatial, TestDocSet, ManagedSchemaRoundRobinCloudTest, TestRequestStatusCollectionAPI, TestSolrQueryResponse, SuggesterFSTTest, TestCloudJSONFacetJoinDomain, ParsingFieldUpdateProcessorsTest, TestJsonFacets, QueryEqualityTest, TestConfigSetsAPIZkFailure, AddBlockUpdateTest, TestDocumentBuilder, TestSolrCoreSnapshots, TestCollectionAPI, TestCollectionAPIs, LeaderInitiatedRecoveryOnShardRestartTest, CloudExitableDirectoryReaderTest, TestCustomStream, TestSearchPerf, LeaderFailureAfterFreshStartTest, HdfsLockFactoryTest, TestLocalFSCloudBackupRestore, DistributedTermsComponentTest, ShardRoutingCustomTest, CdcrRequestHandlerTest, SpellPossibilityIteratorTest, TestQueryTypes, TestSerializedLuceneMatchVersion, DeleteReplicaTest, SolrCLIZkUtilsTest, PingRequestHandlerTest, LukeRequestHandlerTest, TestConfigSets, TestFastWriter, TestLegacyFieldReuse, TestReplicaProperties, TestRestoreCore, TestPullReplicaErrorHandling, TestCoreAdminApis, TestCoreDiscovery, TestFastLRUCache, HLLSerializationTest, LoggingHandlerTest, CoreMergeIndexesAdminHandlerTest, TestCloudDeleteByQuery, TestHdfsCloudBackupRestore, SolrXmlInZkTest, TestMacros, TestImpersonationWithHadoopAuth, BadComponentTest, TestNumericTerms64, TestSha256AuthenticationProvider, ConvertedLegacyTest, TestSolrCloudWithKerberosAlt, SolrShardReporterTest, ChaosMonkeyNothingIsSafeWithPullReplicasTest, SecurityConfHandlerTest, TestSegmentSorting, SolrSlf4jReporterTest]
   [junit4] Completed [152/732 (1!)] on J1 in 6.40s, 1 test, 1 failure <<< FAILURES!

[...truncated 949 lines...]
   [junit4] Suite: org.apache.solr.update.processor.UpdateRequestProcessorFactoryTest
   [junit4]   2> 1972299 INFO  (SUITE-UpdateRequestProcessorFactoryTest-seed#[AE83759C898BDB1D]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.update.processor.UpdateRequestProcessorFactoryTest_AE83759C898BDB1D-001/init-core-data-001
   [junit4]   2> 1972300 WARN  (SUITE-UpdateRequestProcessorFactoryTest-seed#[AE83759C898BDB1D]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=2 numCloses=2
   [junit4]   2> 1972301 INFO  (SUITE-UpdateRequestProcessorFactoryTest-seed#[AE83759C898BDB1D]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 1972306 INFO  (SUITE-UpdateRequestProcessorFactoryTest-seed#[AE83759C898BDB1D]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN) w/ MAC_OS_X supressed clientAuth
   [junit4]   2> 1972306 INFO  (SUITE-UpdateRequestProcessorFactoryTest-seed#[AE83759C898BDB1D]-worker) [    ] o.a.s.SolrTestCaseJ4 ####initCore
   [junit4]   2> 1973614 WARN  (coreLoadExecutor-8349-thread-1) [    x:collection1] o.a.s.c.RequestHandlers no default request handler is registered (either '/select' or 'standard')
   [junit4]   2> 1973616 WARN  (coreLoadExecutor-8349-thread-1) [    x:collection1] o.a.s.r.ManagedResourceStorage Cannot write to config directory /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr/collection1/conf; switching to use InMemory storage instead.
   [junit4]   2> 1973618 INFO  (SUITE-UpdateRequestProcessorFactoryTest-seed#[AE83759C898BDB1D]-worker) [    ] o.a.s.SolrTestCaseJ4 ####initCore end
   [junit4]   2> 1973629 INFO  (TEST-UpdateRequestProcessorFactoryTest.testRequestTimeUrp-seed#[AE83759C898BDB1D]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testRequestTimeUrp
   [junit4]   2> 1973633 INFO  (TEST-UpdateRequestProcessorFactoryTest.testRequestTimeUrp-seed#[AE83759C898BDB1D]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testRequestTimeUrp
   [junit4]   2> 1973646 INFO  (TEST-UpdateRequestProcessorFactoryTest.testUpdateDistribChainSkipping-seed#[AE83759C898BDB1D]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testUpdateDistribChainSkipping
   [junit4]   2> 1973647 INFO  (TEST-UpdateRequestProcessorFactoryTest.testUpdateDistribChainSkipping-seed#[AE83759C898BDB1D]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testUpdateDistribChainSkipping
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=UpdateRequestProcessorFactoryTest -Dtests.method=testUpdateDistribChainSkipping -Dtests.seed=AE83759C898BDB1D -Dtests.slow=true -Dtests.locale=vi -Dtests.timezone=Mexico/BajaSur -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] FAILURE 0.01s J1 | UpdateRequestProcessorFactoryTest.testUpdateDistribChainSkipping <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: Tests must be run with INFO level logging otherwise LogUpdateProcessor isn't used and can't be tested.
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([AE83759C898BDB1D:DF678B4AFF60FCD1]:0)
   [junit4]    > 	at org.apache.solr.update.processor.UpdateRequestProcessorFactoryTest.testUpdateDistribChainSkipping(UpdateRequestProcessorFactoryTest.java:101)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1973661 INFO  (TEST-UpdateRequestProcessorFactoryTest.testConfiguration-seed#[AE83759C898BDB1D]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testConfiguration
   [junit4]   2> 1973662 INFO  (TEST-UpdateRequestProcessorFactoryTest.testConfiguration-seed#[AE83759C898BDB1D]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testConfiguration
   [junit4]   2> 1973662 INFO  (SUITE-UpdateRequestProcessorFactoryTest-seed#[AE83759C898BDB1D]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J1/temp/solr.update.processor.UpdateRequestProcessorFactoryTest_AE83759C898BDB1D-001
   [junit4]   2> NOTE: test params are: codec=FastDecompressionCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=FAST_DECOMPRESSION, chunkSize=32489, maxDocsPerChunk=4, blockSize=772), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=FAST_DECOMPRESSION, chunkSize=32489, blockSize=772)), sim=RandomSimilarity(queryNorm=true): {}, locale=vi, timezone=Mexico/BajaSur
   [junit4]   2> NOTE: Mac OS X 10.11.6 x86_64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=3,threads=1,free=194076128,total=531103744
   [junit4]   2> NOTE: All tests run in this JVM: [TestRandomRequestDistribution, BJQParserTest, TestSolrXml, TestFiltering, TestLeaderElectionZkExpiry, JvmMetricsTest, TestSuggestSpellingConverter, TestReloadAndDeleteDocs, ClassificationUpdateProcessorFactoryTest, FileUtilsTest, JavabinLoaderTest, TestSubQueryTransformerDistrib, ExplicitHLLTest, TestConfig, TestClassicSimilarityFactory, TestQueryUtils, TestRecoveryHdfs, DocumentAnalysisRequestHandlerTest, TestIndexSearcher, AutoCommitTest, TlogReplayBufferedWhileIndexingTest, ResponseHeaderTest, BigEndianAscendingWordSerializerTest, DocValuesMissingTest, SSLMigrationTest, DistributedMLTComponentTest, ExternalFileFieldSortTest, AtomicUpdateProcessorFactoryTest, TestInPlaceUpdatesStandalone, TestSolr4Spatial, TestDocSet, ManagedSchemaRoundRobinCloudTest, TestRequestStatusCollectionAPI, TestSolrQueryResponse, SuggesterFSTTest, TestCloudJSONFacetJoinDomain, ParsingFieldUpdateProcessorsTest, TestJsonFacets, QueryEqualityTest, TestConfigSetsAPIZkFailure, AddBlockUpdateTest, TestDocumentBuilder, TestSolrCoreSnapshots, TestCollectionAPI, TestCollectionAPIs, LeaderInitiatedRecoveryOnShardRestartTest, CloudExitableDirectoryReaderTest, TestCustomStream, TestSearchPerf, LeaderFailureAfterFreshStartTest, HdfsLockFactoryTest, TestLocalFSCloudBackupRestore, DistributedTermsComponentTest, ShardRoutingCustomTest, CdcrRequestHandlerTest, SpellPossibilityIteratorTest, TestQueryTypes, TestSerializedLuceneMatchVersion, DeleteReplicaTest, SolrCLIZkUtilsTest, PingRequestHandlerTest, LukeRequestHandlerTest, TestConfigSets, TestFastWriter, TestLegacyFieldReuse, TestReplicaProperties, TestRestoreCore, TestPullReplicaErrorHandling, TestCoreAdminApis, TestCoreDiscovery, TestFastLRUCache, HLLSerializationTest, LoggingHandlerTest, CoreMergeIndexesAdminHandlerTest, TestCloudDeleteByQuery, TestHdfsCloudBackupRestore, SolrXmlInZkTest, TestMacros, TestImpersonationWithHadoopAuth, BadComponentTest, TestNumericTerms64, TestSha256AuthenticationProvider, ConvertedLegacyTest, TestSolrCloudWithKerberosAlt, SolrShardReporterTest, ChaosMonkeyNothingIsSafeWithPullReplicasTest, SecurityConfHandlerTest, TestSegmentSorting, SolrSlf4jReporterTest, TestHdfsUpdateLog, LargeFieldTest, TestLuceneMatchVersion, PreAnalyzedFieldTest, DebugComponentTest, SynonymTokenizerTest, TestSchemaSimilarityResource, TestAddFieldRealTimeGet, TestAnalyzedSuggestions, HealthCheckHandlerTest, AutoScalingHandlerTest, TestUseDocValuesAsStored2, FieldMutatingUpdateProcessorTest, TestFilteredDocIdSet, FileBasedSpellCheckerTest, TestDynamicFieldCollectionResource, TestJoin, DistributedQueryComponentOptimizationTest, TestSimpleTextCodec, BasicAuthIntegrationTest, SOLR749Test, SharedFSAutoReplicaFailoverUtilsTest, TestReload, TestLegacyFieldCache, RecoveryAfterSoftCommitTest, TestConfigReload, TestCloudSearcherWarming, MetricsHandlerTest, AssignTest, CdcrUpdateLogTest, TestSolrDeletionPolicy1, TestPayloadCheckQParserPlugin, BufferStoreTest, OpenCloseCoreStressTest, OverseerCollectionConfigSetProcessorTest, MBeansHandlerTest, TestHalfAndHalfDocValues, TestFieldCacheSort, TestFieldTypeCollectionResource, DistributedQueueTest, OverseerRolesTest, IndexSchemaRuntimeFieldTest, TestCustomSort, CdcrReplicationDistributedZkTest, OverseerTaskQueueTest, TestJavabinTupleStreamParser, TestPerFieldSimilarity, TestStressCloudBlindAtomicUpdates, StatsComponentTest, TestTolerantUpdateProcessorCloud, TestMissingGroups, TestCrossCoreJoin, MoreLikeThisHandlerTest, TestDownShardTolerantSearch, DirectSolrSpellCheckerTest, LeaderElectionContextKeyTest, FullSolrCloudDistribCmdsTest, TestDocBasedVersionConstraints, TestRangeQuery, ConcurrentDeleteAndCreateCollectionTest, TestUnifiedSolrHighlighter, TestConfigSetsAPI, SolrGangliaReporterTest, HdfsCollectionsAPIDistributedZkTest, MoveReplicaHDFSUlogDirTest, TermVectorComponentTest, DistributedExpandComponentTest, HdfsUnloadDistributedZkTest, TestSubQueryTransformer, PolyFieldTest, CustomCollectionTest, CollectionsAPIDistributedZkTest, ConfigureRecoveryStrategyTest, StatsReloadRaceTest, TestGeoJSONResponseWriter, TestLegacyTerms, BasicFunctionalityTest, TestSmileRequest, TestMergePolicyConfig, TestStandardQParsers, TestTrie, CollectionsAPIAsyncDistributedZkTest, TestSolrIndexConfig, TestDeleteCollectionOnDownNodes, ZkStateReaderTest, BasicDistributedZkTest, ChaosMonkeySafeLeaderTest, BasicDistributedZk2Test, LeaderElectionIntegrationTest, ShardRoutingTest, BasicZkTest, TestReplicationHandler, TestRandomFaceting, ZkSolrClientTest, TestZkChroot, TestRandomDVFaceting, ZkCLITest, TestDistributedGrouping, TestFaceting, TestRecovery, TestHashPartitioner, DistributedSpellCheckComponentTest, TermVectorComponentDistributedTest, ZkControllerTest, TestRealTimeGet, TestStressReorder, TestCoreContainer, SpellCheckComponentTest, SolrCmdDistributorTest, QueryElevationComponentTest, BadIndexSchemaTest, TestFunctionQuery, TestBadConfig, HighlighterTest, ShowFileRequestHandlerTest, DistributedQueryElevationComponentTest, SolrIndexSplitterTest, SimplePostToolTest, SignatureUpdateProcessorFactoryTest, SpellCheckCollatorTest, SolrRequestParserTest, SuggesterTSTTest, TestStressLucene, SpatialFilterTest, SuggesterWFSTTest, TestCSVLoader, NoCacheHeaderTest, SchemaVersionSpecificBehaviorTest, SolrCoreCheckLockOnStartupTest, TestPseudoReturnFields, TestAtomicUpdateErrorCases, TestWordDelimiterFilterFactory, DefaultValueUpdateProcessorTest, StatelessScriptUpdateProcessorFactoryTest, TestRemoteStreaming, DocValuesMultiTest, DistanceFunctionTest, XsltUpdateRequestHandlerTest, CacheHeaderTest, IndexBasedSpellCheckerTest, TestSurroundQueryParser, DisMaxRequestHandlerTest, TestReversedWildcardFilterFactory, TermsComponentTest, DocumentBuilderTest, TestValueSourceCache, RequiredFieldsTest, TestSolrQueryParser, RegexBoostProcessorTest, SolrPluginUtilsTest, IndexSchemaTest, TestCollationField, UpdateRequestProcessorFactoryTest]
   [junit4] Completed [430/732 (2!)] on J1 in 1.40s, 3 tests, 1 failure <<< FAILURES!

[...truncated 44358 lines...]

[JENKINS] Lucene-Solr-master-MacOSX (64bit/jdk1.8.0) - Build # 4148 - Failure!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-MacOSX/4148/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseConcMarkSweepGC

All tests passed

Build Log:
[...truncated 1536 lines...]
# A fatal error has been detected by the Java Runtime Environment:
#
#  SIGFPE (0x8) at pc=0x00007fff908cb143, pid=62637, tid=0x0000000000004603
#
# JRE version: Java(TM) SE Runtime Environment (8.0_144-b01) (build 1.8.0_144-b01)
# Java VM: Java HotSpot(TM) 64-Bit Server VM (25.144-b01 mixed mode bsd-amd64 compressed oops)
# Problematic frame:
# C  [libsystem_kernel.dylib+0x11143]  __commpage_gettimeofday+0x43
#
# Failed to write core dump. Core dumps have been disabled. To enable core dumping, try "ulimit -c unlimited" before starting Java again
#
# An error report file with more information is saved as:
# /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/hs_err_pid62637.log
#
# If you would like to submit a bug report, please visit:
#   http://bugreport.java.com/bugreport/crash.jsp
#
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any

[JENKINS] Lucene-Solr-master-MacOSX (64bit/jdk1.8.0) - Build # 4147 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-MacOSX/4147/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseParallelGC

8 tests failed.
FAILED:  org.apache.solr.TestDistributedMissingSort.test

Error Message:
Failed to list contents of /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr

Stack Trace:
java.io.IOException: Failed to list contents of /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/core/src/test-files/solr
	at __randomizedtesting.SeedInfo.seed([383E8919793F074B:B06AB6C3D7C36AB3]:0)
	at org.apache.commons.io.FileUtils.doCopyDirectory(FileUtils.java:1426)
	at org.apache.commons.io.FileUtils.copyDirectory(FileUtils.java:1388)
	at org.apache.commons.io.FileUtils.copyDirectory(FileUtils.java:1268)
	at org.apache.commons.io.FileUtils.copyDirectory(FileUtils.java:1237)
	at org.apache.solr.BaseDistributedSearchTestCase.seedSolrHome(BaseDistributedSearchTestCase.java:1100)
	at org.apache.solr.BaseDistributedSearchTestCase.createServers(BaseDistributedSearchTestCase.java:337)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsRepeatStatement.callStatement(BaseDistributedSearchTestCase.java:1017)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.CollectionsAPIAsyncDistributedZkTest

Error Message:
java.io.IOException: Couldn't instantiate org.apache.zookeeper.ClientCnxnSocketNIO

Stack Trace:
org.apache.solr.common.SolrException: java.io.IOException: Couldn't instantiate org.apache.zookeeper.ClientCnxnSocketNIO
	at __randomizedtesting.SeedInfo.seed([383E8919793F074B]:0)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:171)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:117)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:112)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:99)
	at org.apache.solr.cloud.MiniSolrCloudCluster.waitForAllNodes(MiniSolrCloudCluster.java:268)
	at org.apache.solr.cloud.MiniSolrCloudCluster.<init>(MiniSolrCloudCluster.java:262)
	at org.apache.solr.cloud.SolrCloudTestCase$Builder.configure(SolrCloudTestCase.java:190)
	at org.apache.solr.cloud.CollectionsAPIAsyncDistributedZkTest.setupCluster(CollectionsAPIAsyncDistributedZkTest.java:47)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:847)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.io.IOException: Couldn't instantiate org.apache.zookeeper.ClientCnxnSocketNIO
	at org.apache.zookeeper.ZooKeeper.getClientCnxnSocket(ZooKeeper.java:1842)
	at org.apache.zookeeper.ZooKeeper.<init>(ZooKeeper.java:447)
	at org.apache.zookeeper.ZooKeeper.<init>(ZooKeeper.java:380)
	at org.apache.solr.common.cloud.SolrZooKeeper.<init>(SolrZooKeeper.java:43)
	at org.apache.solr.common.cloud.ZkClientConnectionStrategy.createSolrZooKeeper(ZkClientConnectionStrategy.java:105)
	at org.apache.solr.common.cloud.DefaultConnectionStrategy.connect(DefaultConnectionStrategy.java:37)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:146)
	... 31 more
Caused by: java.io.IOException: Too many open files
	at sun.nio.ch.IOUtil.makePipe(Native Method)
	at sun.nio.ch.KQueueSelectorImpl.<init>(KQueueSelectorImpl.java:84)
	at sun.nio.ch.KQueueSelectorProvider.openSelector(KQueueSelectorProvider.java:42)
	at java.nio.channels.Selector.open(Selector.java:227)
	at org.apache.zookeeper.ClientCnxnSocketNIO.<init>(ClientCnxnSocketNIO.java:43)
	at sun.reflect.GeneratedConstructorAccessor72.newInstance(Unknown Source)
	at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
	at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
	at java.lang.Class.newInstance(Class.java:442)
	at org.apache.zookeeper.ZooKeeper.getClientCnxnSocket(ZooKeeper.java:1839)
	... 37 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.CollectionsAPIAsyncDistributedZkTest

Error Message:
ObjectTracker found 9 object(s) that were not released!!! [SolrZkClient, InternalHttpClient, Overseer, InternalHttpClient, SolrZkClient, ZkController, ZkController, InternalHttpClient, InternalHttpClient] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.common.cloud.SolrZkClient  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:185)  at org.apache.solr.cloud.ZkController.<init>(ZkController.java:311)  at org.apache.solr.core.ZkContainer.initZooKeeper(ZkContainer.java:112)  at org.apache.solr.core.CoreContainer.load(CoreContainer.java:518)  at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:263)  at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:183)  at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:139)  at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873)  at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1596)  at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1659)  at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1316)  at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1145)  at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306)  at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)  at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367)  at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:384)  at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:247)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:289)  at org.apache.solr.handler.component.HttpShardHandlerFactory.init(HttpShardHandlerFactory.java:209)  at org.apache.solr.handler.component.ShardHandlerFactory.newInstance(ShardHandlerFactory.java:47)  at org.apache.solr.core.CoreContainer.load(CoreContainer.java:502)  at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:263)  at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:183)  at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:139)  at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873)  at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1596)  at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1659)  at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1316)  at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1145)  at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306)  at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)  at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367)  at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:384)  at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:247)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.cloud.Overseer  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.cloud.Overseer.start(Overseer.java:534)  at org.apache.solr.cloud.OverseerElectionContext.runLeaderProcess(ElectionContext.java:784)  at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170)  at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135)  at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:307)  at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:216)  at org.apache.solr.cloud.ZkController.init(ZkController.java:748)  at org.apache.solr.cloud.ZkController.<init>(ZkController.java:435)  at org.apache.solr.core.ZkContainer.initZooKeeper(ZkContainer.java:112)  at org.apache.solr.core.CoreContainer.load(CoreContainer.java:518)  at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:263)  at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:183)  at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:139)  at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873)  at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1596)  at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1659)  at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1316)  at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1145)  at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306)  at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)  at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367)  at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:384)  at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:247)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:289)  at org.apache.solr.update.UpdateShardHandler.<init>(UpdateShardHandler.java:94)  at org.apache.solr.core.CoreContainer.load(CoreContainer.java:508)  at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:263)  at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:183)  at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:139)  at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873)  at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1596)  at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1659)  at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1316)  at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1145)  at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306)  at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)  at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367)  at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:384)  at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:247)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.common.cloud.SolrZkClient  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:185)  at org.apache.solr.cloud.ZkController.<init>(ZkController.java:311)  at org.apache.solr.core.ZkContainer.initZooKeeper(ZkContainer.java:112)  at org.apache.solr.core.CoreContainer.load(CoreContainer.java:518)  at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:263)  at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:183)  at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:139)  at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873)  at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1596)  at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1659)  at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1316)  at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1145)  at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306)  at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)  at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367)  at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:384)  at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:247)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.cloud.ZkController  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.cloud.ZkController.<init>(ZkController.java:437)  at org.apache.solr.core.ZkContainer.initZooKeeper(ZkContainer.java:112)  at org.apache.solr.core.CoreContainer.load(CoreContainer.java:518)  at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:263)  at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:183)  at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:139)  at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873)  at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1596)  at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1659)  at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1316)  at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1145)  at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306)  at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)  at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367)  at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:384)  at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:247)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.cloud.ZkController  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.cloud.ZkController.<init>(ZkController.java:437)  at org.apache.solr.core.ZkContainer.initZooKeeper(ZkContainer.java:112)  at org.apache.solr.core.CoreContainer.load(CoreContainer.java:518)  at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:263)  at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:183)  at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:139)  at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873)  at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1596)  at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1659)  at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1316)  at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1145)  at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306)  at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)  at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367)  at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:384)  at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:247)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:289)  at org.apache.solr.update.UpdateShardHandler.<init>(UpdateShardHandler.java:94)  at org.apache.solr.core.CoreContainer.load(CoreContainer.java:508)  at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:263)  at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:183)  at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:139)  at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873)  at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1596)  at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1659)  at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1316)  at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1145)  at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306)  at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)  at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367)  at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:384)  at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:247)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:289)  at org.apache.solr.handler.component.HttpShardHandlerFactory.init(HttpShardHandlerFactory.java:209)  at org.apache.solr.handler.component.ShardHandlerFactory.newInstance(ShardHandlerFactory.java:47)  at org.apache.solr.core.CoreContainer.load(CoreContainer.java:502)  at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:263)  at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:183)  at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:139)  at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873)  at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1596)  at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1659)  at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1316)  at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1145)  at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306)  at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)  at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394)  at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367)  at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:384)  at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:247)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  

Stack Trace:
java.lang.AssertionError: ObjectTracker found 9 object(s) that were not released!!! [SolrZkClient, InternalHttpClient, Overseer, InternalHttpClient, SolrZkClient, ZkController, ZkController, InternalHttpClient, InternalHttpClient]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.common.cloud.SolrZkClient
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:185)
	at org.apache.solr.cloud.ZkController.<init>(ZkController.java:311)
	at org.apache.solr.core.ZkContainer.initZooKeeper(ZkContainer.java:112)
	at org.apache.solr.core.CoreContainer.load(CoreContainer.java:518)
	at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:263)
	at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:183)
	at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:139)
	at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873)
	at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1596)
	at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1659)
	at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1316)
	at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1145)
	at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367)
	at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:384)
	at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:247)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:289)
	at org.apache.solr.handler.component.HttpShardHandlerFactory.init(HttpShardHandlerFactory.java:209)
	at org.apache.solr.handler.component.ShardHandlerFactory.newInstance(ShardHandlerFactory.java:47)
	at org.apache.solr.core.CoreContainer.load(CoreContainer.java:502)
	at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:263)
	at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:183)
	at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:139)
	at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873)
	at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1596)
	at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1659)
	at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1316)
	at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1145)
	at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367)
	at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:384)
	at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:247)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.cloud.Overseer
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
	at org.apache.solr.cloud.Overseer.start(Overseer.java:534)
	at org.apache.solr.cloud.OverseerElectionContext.runLeaderProcess(ElectionContext.java:784)
	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170)
	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135)
	at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:307)
	at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:216)
	at org.apache.solr.cloud.ZkController.init(ZkController.java:748)
	at org.apache.solr.cloud.ZkController.<init>(ZkController.java:435)
	at org.apache.solr.core.ZkContainer.initZooKeeper(ZkContainer.java:112)
	at org.apache.solr.core.CoreContainer.load(CoreContainer.java:518)
	at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:263)
	at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:183)
	at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:139)
	at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873)
	at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1596)
	at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1659)
	at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1316)
	at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1145)
	at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367)
	at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:384)
	at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:247)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:289)
	at org.apache.solr.update.UpdateShardHandler.<init>(UpdateShardHandler.java:94)
	at org.apache.solr.core.CoreContainer.load(CoreContainer.java:508)
	at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:263)
	at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:183)
	at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:139)
	at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873)
	at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1596)
	at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1659)
	at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1316)
	at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1145)
	at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367)
	at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:384)
	at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:247)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.common.cloud.SolrZkClient
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
	at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:185)
	at org.apache.solr.cloud.ZkController.<init>(ZkController.java:311)
	at org.apache.solr.core.ZkContainer.initZooKeeper(ZkContainer.java:112)
	at org.apache.solr.core.CoreContainer.load(CoreContainer.java:518)
	at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:263)
	at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:183)
	at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:139)
	at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873)
	at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1596)
	at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1659)
	at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1316)
	at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1145)
	at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367)
	at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:384)
	at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:247)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.cloud.ZkController
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
	at org.apache.solr.cloud.ZkController.<init>(ZkController.java:437)
	at org.apache.solr.core.ZkContainer.initZooKeeper(ZkContainer.java:112)
	at org.apache.solr.core.CoreContainer.load(CoreContainer.java:518)
	at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:263)
	at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:183)
	at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:139)
	at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873)
	at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1596)
	at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1659)
	at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1316)
	at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1145)
	at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367)
	at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:384)
	at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:247)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.cloud.ZkController
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
	at org.apache.solr.cloud.ZkController.<init>(ZkController.java:437)
	at org.apache.solr.core.ZkContainer.initZooKeeper(ZkContainer.java:112)
	at org.apache.solr.core.CoreContainer.load(CoreContainer.java:518)
	at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:263)
	at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:183)
	at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:139)
	at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873)
	at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1596)
	at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1659)
	at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1316)
	at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1145)
	at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367)
	at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:384)
	at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:247)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:289)
	at org.apache.solr.update.UpdateShardHandler.<init>(UpdateShardHandler.java:94)
	at org.apache.solr.core.CoreContainer.load(CoreContainer.java:508)
	at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:263)
	at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:183)
	at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:139)
	at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873)
	at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1596)
	at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1659)
	at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1316)
	at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1145)
	at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367)
	at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:384)
	at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:247)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:289)
	at org.apache.solr.handler.component.HttpShardHandlerFactory.init(HttpShardHandlerFactory.java:209)
	at org.apache.solr.handler.component.ShardHandlerFactory.newInstance(ShardHandlerFactory.java:47)
	at org.apache.solr.core.CoreContainer.load(CoreContainer.java:502)
	at org.apache.solr.servlet.SolrDispatchFilter.createCoreContainer(SolrDispatchFilter.java:263)
	at org.apache.solr.servlet.SolrDispatchFilter.init(SolrDispatchFilter.java:183)
	at org.eclipse.jetty.servlet.FilterHolder.initialize(FilterHolder.java:139)
	at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:873)
	at org.eclipse.jetty.servlet.ServletHandler.updateMappings(ServletHandler.java:1596)
	at org.eclipse.jetty.servlet.ServletHandler.setFilterMappings(ServletHandler.java:1659)
	at org.eclipse.jetty.servlet.ServletHandler.addFilterMapping(ServletHandler.java:1316)
	at org.eclipse.jetty.servlet.ServletHandler.addFilterWithMapping(ServletHandler.java:1145)
	at org.eclipse.jetty.servlet.ServletContextHandler.addFilter(ServletContextHandler.java:448)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$1.lifeCycleStarted(JettySolrRunner.java:306)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.setStarted(AbstractLifeCycle.java:179)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:69)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:394)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:367)
	at org.apache.solr.cloud.MiniSolrCloudCluster.startJettySolrRunner(MiniSolrCloudCluster.java:384)
	at org.apache.solr.cloud.MiniSolrCloudCluster.lambda$new$0(MiniSolrCloudCluster.java:247)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)


	at __randomizedtesting.SeedInfo.seed([383E8919793F074B]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNull(Assert.java:551)
	at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:291)
	at sun.reflect.GeneratedMethodAccessor50.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:870)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.CollectionsAPIAsyncDistributedZkTest

Error Message:
37 threads leaked from SUITE scope at org.apache.solr.cloud.CollectionsAPIAsyncDistributedZkTest:     1) Thread[id=9695, name=OverseerHdfsCoreFailoverThread-98549145225461765-127.0.0.1:54355_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:139)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=9670, name=org.eclipse.jetty.server.session.HashSessionManager@44f155cfTimer, state=TIMED_WAITING, group=TGRP-CollectionsAPIAsyncDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=9692, name=zkCallback-1550-thread-1, state=TIMED_WAITING, group=TGRP-CollectionsAPIAsyncDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    4) Thread[id=9653, name=qtp1663731454-9653, state=RUNNABLE, group=TGRP-CollectionsAPIAsyncDistributedZkTest]         at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)         at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)         at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)         at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)         at java.lang.Thread.run(Thread.java:748)    5) Thread[id=9667, name=org.eclipse.jetty.server.session.HashSessionManager@168eb359Timer, state=TIMED_WAITING, group=TGRP-CollectionsAPIAsyncDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    6) Thread[id=9658, name=qtp728474246-9658, state=RUNNABLE, group=TGRP-CollectionsAPIAsyncDistributedZkTest]         at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)         at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)         at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)         at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)         at java.lang.Thread.run(Thread.java:748)    7) Thread[id=9661, name=qtp728474246-9661, state=TIMED_WAITING, group=TGRP-CollectionsAPIAsyncDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)    8) Thread[id=9687, name=jetty-launcher-1538-thread-1-SendThread(127.0.0.1:54349), state=RUNNABLE, group=TGRP-CollectionsAPIAsyncDistributedZkTest]         at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)         at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)         at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1141)    9) Thread[id=9643, name=Thread-1726, state=WAITING, group=TGRP-CollectionsAPIAsyncDistributedZkTest]         at java.lang.Object.wait(Native Method)         at java.lang.Thread.join(Thread.java:1252)         at java.lang.Thread.join(Thread.java:1326)         at org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:297)         at org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:309)         at org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:490)   10) Thread[id=9679, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CollectionsAPIAsyncDistributedZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)   11) Thread[id=9647, name=ProcessThread(sid:0 cport:54349):, state=WAITING, group=TGRP-CollectionsAPIAsyncDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:122)   12) Thread[id=9669, name=qtp728474246-9669, state=TIMED_WAITING, group=TGRP-CollectionsAPIAsyncDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   13) Thread[id=9668, name=qtp728474246-9668, state=TIMED_WAITING, group=TGRP-CollectionsAPIAsyncDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   14) Thread[id=9644, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-CollectionsAPIAsyncDistributedZkTest]         at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)         at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)         at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:173)         at java.lang.Thread.run(Thread.java:748)   15) Thread[id=9656, name=qtp1663731454-9656, state=TIMED_WAITING, group=TGRP-CollectionsAPIAsyncDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   16) Thread[id=9677, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CollectionsAPIAsyncDistributedZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)   17) Thread[id=9690, name=jetty-launcher-1538-thread-2-EventThread, state=WAITING, group=TGRP-CollectionsAPIAsyncDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)   18) Thread[id=9659, name=qtp728474246-9659-acceptor-0@76b0892c-ServerConnector@6237addc{HTTP/1.1,[http/1.1]}{127.0.0.1:54355}, state=RUNNABLE, group=TGRP-CollectionsAPIAsyncDistributedZkTest]         at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)         at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371)         at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)         at java.lang.Thread.run(Thread.java:748)   19) Thread[id=9660, name=qtp1663731454-9660, state=TIMED_WAITING, group=TGRP-CollectionsAPIAsyncDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   20) Thread[id=9664, name=qtp728474246-9664, state=TIMED_WAITING, group=TGRP-CollectionsAPIAsyncDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   21) Thread[id=9657, name=qtp1663731454-9657, state=TIMED_WAITING, group=TGRP-CollectionsAPIAsyncDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   22) Thread[id=9655, name=qtp1663731454-9655, state=TIMED_WAITING, group=TGRP-CollectionsAPIAsyncDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   23) Thread[id=9663, name=qtp1663731454-9663, state=TIMED_WAITING, group=TGRP-CollectionsAPIAsyncDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   24) Thread[id=9689, name=jetty-launcher-1538-thread-2-SendThread(127.0.0.1:54349), state=RUNNABLE, group=TGRP-CollectionsAPIAsyncDistributedZkTest]         at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)         at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)         at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1141)   25) Thread[id=9691, name=zkCallback-1549-thread-1, state=TIMED_WAITING, group=TGRP-CollectionsAPIAsyncDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   26) Thread[id=9645, name=SessionTracker, state=TIMED_WAITING, group=TGRP-CollectionsAPIAsyncDistributedZkTest]         at java.lang.Object.wait(Native Method)         at org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:146)   27) Thread[id=9652, name=jetty-launcher-1538-thread-2, state=TIMED_WAITING, group=TGRP-CollectionsAPIAsyncDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   28) Thread[id=9678, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CollectionsAPIAsyncDistributedZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)   29) Thread[id=9662, name=qtp728474246-9662, state=TIMED_WAITING, group=TGRP-CollectionsAPIAsyncDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   30) Thread[id=9651, name=jetty-launcher-1538-thread-1, state=TIMED_WAITING, group=TGRP-CollectionsAPIAsyncDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   31) Thread[id=9666, name=qtp728474246-9666, state=TIMED_WAITING, group=TGRP-CollectionsAPIAsyncDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   32) Thread[id=9693, name=OverseerStateUpdate-98549145225461765-127.0.0.1:54355_solr-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:165)         at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:139)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:184)         at java.lang.Thread.run(Thread.java:748)   33) Thread[id=9694, name=OverseerCollectionConfigSetProcessor-98549145225461765-127.0.0.1:54355_solr-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.apache.solr.cloud.DistributedQueue.peekElements(DistributedQueue.java:357)         at org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:235)         at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:228)         at java.lang.Thread.run(Thread.java:748)   34) Thread[id=9680, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CollectionsAPIAsyncDistributedZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)   35) Thread[id=9654, name=qtp1663731454-9654-acceptor-0@4884df92-ServerConnector@3ca85fb0{HTTP/1.1,[http/1.1]}{127.0.0.1:54356}, state=RUNNABLE, group=TGRP-CollectionsAPIAsyncDistributedZkTest]         at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)         at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371)         at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)         at java.lang.Thread.run(Thread.java:748)   36) Thread[id=9665, name=qtp1663731454-9665, state=TIMED_WAITING, group=TGRP-CollectionsAPIAsyncDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   37) Thread[id=9688, name=jetty-launcher-1538-thread-1-EventThread, state=WAITING, group=TGRP-CollectionsAPIAsyncDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 37 threads leaked from SUITE scope at org.apache.solr.cloud.CollectionsAPIAsyncDistributedZkTest: 
   1) Thread[id=9695, name=OverseerHdfsCoreFailoverThread-98549145225461765-127.0.0.1:54355_solr-n_0000000000, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:139)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=9670, name=org.eclipse.jetty.server.session.HashSessionManager@44f155cfTimer, state=TIMED_WAITING, group=TGRP-CollectionsAPIAsyncDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=9692, name=zkCallback-1550-thread-1, state=TIMED_WAITING, group=TGRP-CollectionsAPIAsyncDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   4) Thread[id=9653, name=qtp1663731454-9653, state=RUNNABLE, group=TGRP-CollectionsAPIAsyncDistributedZkTest]
        at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
        at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)
        at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
        at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
        at java.lang.Thread.run(Thread.java:748)
   5) Thread[id=9667, name=org.eclipse.jetty.server.session.HashSessionManager@168eb359Timer, state=TIMED_WAITING, group=TGRP-CollectionsAPIAsyncDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   6) Thread[id=9658, name=qtp728474246-9658, state=RUNNABLE, group=TGRP-CollectionsAPIAsyncDistributedZkTest]
        at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
        at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)
        at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
        at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
        at java.lang.Thread.run(Thread.java:748)
   7) Thread[id=9661, name=qtp728474246-9661, state=TIMED_WAITING, group=TGRP-CollectionsAPIAsyncDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
   8) Thread[id=9687, name=jetty-launcher-1538-thread-1-SendThread(127.0.0.1:54349), state=RUNNABLE, group=TGRP-CollectionsAPIAsyncDistributedZkTest]
        at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
        at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)
        at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1141)
   9) Thread[id=9643, name=Thread-1726, state=WAITING, group=TGRP-CollectionsAPIAsyncDistributedZkTest]
        at java.lang.Object.wait(Native Method)
        at java.lang.Thread.join(Thread.java:1252)
        at java.lang.Thread.join(Thread.java:1326)
        at org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:297)
        at org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:309)
        at org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:490)
  10) Thread[id=9679, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CollectionsAPIAsyncDistributedZkTest]
        at java.lang.Thread

[...truncated too long message...]

ume.java:148)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    2) Thread[id=9689, name=jetty-launcher-1538-thread-2-SendThread(127.0.0.1:54349), state=TIMED_WAITING, group=TGRP-CollectionsAPIAsyncDistributedZkTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)
   [junit4]    >    3) Thread[id=9691, name=zkCallback-1549-thread-1, state=TIMED_WAITING, group=TGRP-CollectionsAPIAsyncDistributedZkTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    4) Thread[id=9687, name=jetty-launcher-1538-thread-1-SendThread(127.0.0.1:54349), state=TIMED_WAITING, group=TGRP-CollectionsAPIAsyncDistributedZkTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
   [junit4]    >    5) Thread[id=9652, name=jetty-launcher-1538-thread-2, state=TIMED_WAITING, group=TGRP-CollectionsAPIAsyncDistributedZkTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    6) Thread[id=9670, name=org.eclipse.jetty.server.session.HashSessionManager@44f155cfTimer, state=TIMED_WAITING, group=TGRP-CollectionsAPIAsyncDistributedZkTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]    >         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
   [junit4]    >         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    7) Thread[id=9692, name=zkCallback-1550-thread-1, state=TIMED_WAITING, group=TGRP-CollectionsAPIAsyncDistributedZkTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    8) Thread[id=9653, name=qtp1663731454-9653, state=RUNNABLE, group=TGRP-CollectionsAPIAsyncDistributedZkTest]
   [junit4]    >         at sun.nio.ch.KQueueArrayWrapper.interrupt(Native Method)
   [junit4]    >         at sun.nio.ch.KQueueArrayWrapper.interrupt(KQueueArrayWrapper.java:203)
   [junit4]    >         at sun.nio.ch.KQueueSelectorImpl.wakeup(KQueueSelectorImpl.java:257)
   [junit4]    >         at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)
   [junit4]    >         at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)
   [junit4]    >         at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:116)
   [junit4]    >         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    9) Thread[id=9667, name=org.eclipse.jetty.server.session.HashSessionManager@168eb359Timer, state=TIMED_WAITING, group=TGRP-CollectionsAPIAsyncDistributedZkTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]    >         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
   [junit4]    >         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >   10) Thread[id=9651, name=jetty-launcher-1538-thread-1, state=TIMED_WAITING, group=TGRP-CollectionsAPIAsyncDistributedZkTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([383E8919793F074B]:0)
   [junit4] Completed [294/732 (5!)] on J1 in 46.10s, 0 tests, 1 failure, 3 errors <<< FAILURES!

[...truncated 44763 lines...]

[JENKINS] Lucene-Solr-master-MacOSX (64bit/jdk1.8.0) - Build # 4146 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-MacOSX/4146/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseParallelGC

4 tests failed.
FAILED:  org.apache.solr.cloud.ForceLeaderTest.testReplicasInLIRNoLeader

Error Message:
Doc with id=1 not found in http://127.0.0.1:65012/ne/k/forceleader_test_collection due to: Path not found: /id; rsp={doc=null}

Stack Trace:
java.lang.AssertionError: Doc with id=1 not found in http://127.0.0.1:65012/ne/k/forceleader_test_collection due to: Path not found: /id; rsp={doc=null}
	at __randomizedtesting.SeedInfo.seed([667BF9D15C701FF8:80ECCD1165F2E699]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.cloud.HttpPartitionTest.assertDocExists(HttpPartitionTest.java:603)
	at org.apache.solr.cloud.HttpPartitionTest.assertDocsExistInAllReplicas(HttpPartitionTest.java:556)
	at org.apache.solr.cloud.ForceLeaderTest.testReplicasInLIRNoLeader(ForceLeaderTest.java:142)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.test

Error Message:
Could not find collection:collection2

Stack Trace:
java.lang.AssertionError: Could not find collection:collection2
	at __randomizedtesting.SeedInfo.seed([667BF9D15C701FF8:EE2FC60BF28C7200]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNotNull(Assert.java:526)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:155)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:140)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:135)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:908)
	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.testIndexingBatchPerRequestWithHttpSolrClient(FullSolrCloudDistribCmdsTest.java:612)
	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.test(FullSolrCloudDistribCmdsTest.java:152)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.update.processor.UpdateRequestProcessorFactoryTest.testUpdateDistribChainSkipping

Error Message:
Tests must be run with INFO level logging otherwise LogUpdateProcessor isn't used and can't be tested.

Stack Trace:
java.lang.AssertionError: Tests must be run with INFO level logging otherwise LogUpdateProcessor isn't used and can't be tested.
	at __randomizedtesting.SeedInfo.seed([667BF9D15C701FF8:179F07072A9B3834]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.update.processor.UpdateRequestProcessorFactoryTest.testUpdateDistribChainSkipping(UpdateRequestProcessorFactoryTest.java:101)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.metrics.reporters.SolrSlf4jReporterTest.testReporter

Error Message:
count1=6, count2=6 - no 'solr.node' logs in: {numFound=0,start=0,docs=[]}

Stack Trace:
java.lang.AssertionError: count1=6, count2=6 - no 'solr.node' logs in: {numFound=0,start=0,docs=[]}
	at __randomizedtesting.SeedInfo.seed([667BF9D15C701FF8:399FD4E6377C8CBD]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.metrics.reporters.SolrSlf4jReporterTest.testReporter(SolrSlf4jReporterTest.java:90)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 11955 lines...]
   [junit4] Suite: org.apache.solr.cloud.FullSolrCloudDistribCmdsTest
   [junit4]   2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.FullSolrCloudDistribCmdsTest_667BF9D15C701FF8-001/init-core-data-001
   [junit4]   2> 1548336 WARN  (SUITE-FullSolrCloudDistribCmdsTest-seed#[667BF9D15C701FF8]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=10 numCloses=10
   [junit4]   2> 1548344 ERROR (Thread-2765) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 1548919 ERROR (TEST-FullSolrCloudDistribCmdsTest.test-seed#[667BF9D15C701FF8]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1550391 WARN  (qtp876805094-15832) [n:127.0.0.1:63605_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 1551641 WARN  (OverseerThreadFactory-6790-thread-2-processing-n:127.0.0.1:63605_) [n:127.0.0.1:63605_    ] o.a.s.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
   [junit4]   2> 1552122 ERROR (TEST-FullSolrCloudDistribCmdsTest.test-seed#[667BF9D15C701FF8]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1553364 WARN  (qtp178589531-15874) [n:127.0.0.1:63617_ c:collection1 s:shard2  x:collection1_shard2_replica_n61] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 1553967 ERROR (TEST-FullSolrCloudDistribCmdsTest.test-seed#[667BF9D15C701FF8]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1555266 WARN  (qtp1355524555-15901) [n:127.0.0.1:63627_ c:collection1 s:shard3  x:collection1_shard3_replica_n63] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 1556392 ERROR (TEST-FullSolrCloudDistribCmdsTest.test-seed#[667BF9D15C701FF8]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1557672 WARN  (qtp561933325-15931) [n:127.0.0.1:63646_ c:collection1 s:shard1  x:collection1_shard1_replica_n65] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 1558143 ERROR (TEST-FullSolrCloudDistribCmdsTest.test-seed#[667BF9D15C701FF8]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1559399 WARN  (qtp1734619594-15959) [n:127.0.0.1:63657_ c:collection1 s:shard2  x:collection1_shard2_replica_n67] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 1559709 ERROR (TEST-FullSolrCloudDistribCmdsTest.test-seed#[667BF9D15C701FF8]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1561024 WARN  (qtp1729832861-15991) [n:127.0.0.1:63679_ c:collection1 s:shard3  x:collection1_shard3_replica_n69] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 1561464 ERROR (TEST-FullSolrCloudDistribCmdsTest.test-seed#[667BF9D15C701FF8]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 1563428 WARN  (qtp948497334-16024) [n:127.0.0.1:63687_ c:collection1 s:shard1  x:collection1_shard1_replica_n71] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 1568159 WARN  (zkCallback-3276-thread-1-processing-n:127.0.0.1:63646_) [n:127.0.0.1:63646_    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 1568172 WARN  (zkCallback-3296-thread-1-processing-n:127.0.0.1:63687_) [n:127.0.0.1:63687_    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 1569162 WARN  (zkCallback-3296-thread-2-processing-n:127.0.0.1:63687_) [n:127.0.0.1:63687_ c:collection1 s:shard1 r:core_node72 x:collection1_shard1_replica_n71] o.a.s.c.SyncStrategy Closed, skipping sync up.
   [junit4]   2> 1574260 ERROR (qtp178589531-15876) [n:127.0.0.1:63617_ c:collection1 s:shard2 r:core_node62 x:collection1_shard2_replica_n61] o.a.s.h.RequestHandlerBase org.apache.solr.common.SolrException: No registered leader was found after waiting for 4000ms , collection: collection1 slice: shard1
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:758)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:744)
   [junit4]   2> 	at org.apache.solr.update.processor.DistributedUpdateProcessor.setupRequest(DistributedUpdateProcessor.java:404)
   [junit4]   2> 	at org.apache.solr.update.processor.DistributedUpdateProcessor.setupRequest(DistributedUpdateProcessor.java:350)
   [junit4]   2> 	at org.apache.solr.update.processor.DistributedUpdateProcessor.processAdd(DistributedUpdateProcessor.java:708)
   [junit4]   2> 	at org.apache.solr.handler.loader.JavabinLoader$1.update(JavabinLoader.java:98)
   [junit4]   2> 	at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readOuterMostDocIterator(JavaBinUpdateRequestCodec.java:188)
   [junit4]   2> 	at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readIterator(JavaBinUpdateRequestCodec.java:144)
   [junit4]   2> 	at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:311)
   [junit4]   2> 	at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:256)
   [junit4]   2> 	at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec$1.readNamedList(JavaBinUpdateRequestCodec.java:130)
   [junit4]   2> 	at org.apache.solr.common.util.JavaBinCodec.readObject(JavaBinCodec.java:276)
   [junit4]   2> 	at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:256)
   [junit4]   2> 	at org.apache.solr.common.util.JavaBinCodec.unmarshal(JavaBinCodec.java:178)
   [junit4]   2> 	at org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec.unmarshal(JavaBinUpdateRequestCodec.java:195)
   [junit4]   2> 	at org.apache.solr.handler.loader.JavabinLoader.parseAndLoadDocs(JavabinLoader.java:108)
   [junit4]   2> 	at org.apache.solr.handler.loader.JavabinLoader.load(JavabinLoader.java:55)
   [junit4]   2> 	at org.apache.solr.handler.UpdateRequestHandler$1.load(UpdateRequestHandler.java:97)
   [junit4]   2> 	at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:68)
   [junit4]   2> 	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.execute(SolrCore.java:2483)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:720)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:526)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 
   [junit4]   2> 1574450 WARN  (zkCallback-3282-thread-2-processing-n:127.0.0.1:63657_) [n:127.0.0.1:63657_    ] o.a.s.c.LeaderElector 
   [junit4]   2> org.apache.solr.common.SolrException: SolrCore not found:collection1_shard2_replica_n67 in []
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:297)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:56)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:348)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$3.lambda$process$0(SolrZkClient.java:269)
   [junit4]   2> 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1574462 WARN  (zkCallback-3289-thread-1-processing-n:127.0.0.1:63679_) [n:127.0.0.1:63679_    ] o.a.s.c.LeaderElector Our node is no longer in line to be leader
   [junit4]   2> 1577058 WARN  (qtp1734619594-15962) [n:127.0.0.1:63657_ c:collection2 s:shard1  x:collection2_shard1_replica_n3] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 1577103 WARN  (qtp1355524555-15904) [n:127.0.0.1:63627_ c:collection2 s:shard3  x:collection2_shard3_replica_n12] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 1577156 WARN  (qtp1729832861-15989) [n:127.0.0.1:63679_ c:collection2 s:shard1  x:collection2_shard1_replica_n2] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 1577259 WARN  (qtp876805094-15828) [n:127.0.0.1:63605_ c:collection2 s:shard1  x:collection2_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 1577282 WARN  (qtp1355524555-15905) [n:127.0.0.1:63627_ c:collection2 s:shard2  x:collection2_shard2_replica_n4] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 1577292 WARN  (qtp1355524555-15903) [n:127.0.0.1:63627_ c:collection2 s:shard5  x:collection2_shard5_replica_n19] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 1577317 WARN  (qtp1355524555-15902) [n:127.0.0.1:63627_ c:collection2 s:shard7  x:collection2_shard7_replica_n27] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 1577326 WARN  (qtp178589531-15873) [n:127.0.0.1:63617_ c:collection2 s:shard4  x:collection2_shard4_replica_n13] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 1577337 WARN  (qtp178589531-15874) [n:127.0.0.1:63617_ c:collection2 s:shard2  x:collection2_shard2_replica_n6] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 1577365 WARN  (qtp876805094-15830) [n:127.0.0.1:63605_ c:collection2 s:shard6  x:collection2_shard6_replica_n22] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 1577371 WARN  (qtp178589531-15870) [n:127.0.0.1:63617_ c:collection2 s:shard7  x:collection2_shard7_replica_n28] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 1577429 WARN  (qtp1729832861-15991) [n:127.0.0.1:63679_ c:collection2 s:shard3  x:collection2_shard3_replica_n9] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 1577486 WARN  (qtp178589531-15872) [n:127.0.0.1:63617_ c:collection2 s:shard5  x:collection2_shard5_replica_n21] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 1577486 WARN  (qtp876805094-15835) [n:127.0.0.1:63605_ c:collection2 s:shard7  x:collection2_shard7_replica_n30] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 1577504 WARN  (qtp1729832861-15994) [n:127.0.0.1:63679_ c:collection2 s:shard4  x:collection2_shard4_replica_n16] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 1577513 WARN  (qtp1729832861-15993) [n:127.0.0.1:63679_ c:collection2 s:shard6  x:collection2_shard6_replica_n24] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 1577513 WARN  (qtp1734619594-15964) [n:127.0.0.1:63657_ c:collection2 s:shard3  x:collection2_shard3_replica_n10] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 1577528 WARN  (qtp1734619594-15957) [n:127.0.0.1:63657_ c:collection2 s:shard5  x:collection2_shard5_replica_n18] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 1577544 WARN  (qtp876805094-15834) [n:127.0.0.1:63605_ c:collection2 s:shard2  x:collection2_shard2_replica_n7] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 1577554 WARN  (qtp876805094-15832) [n:127.0.0.1:63605_ c:collection2 s:shard4  x:collection2_shard4_replica_n15] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 1577637 WARN  (qtp1734619594-15960) [n:127.0.0.1:63657_ c:collection2 s:shard6  x:collection2_shard6_replica_n25] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 1577708 ERROR (qtp1734619594-15960) [n:127.0.0.1:63657_ c:collection2 s:shard6  x:collection2_shard6_replica_n25] o.a.s.h.RequestHandlerBase org.apache.solr.common.SolrException: Error CREATEing SolrCore 'collection2_shard6_replica_n25': Unable to create core [collection2_shard6_replica_n25] Caused by: invalid API spec: apispec/core.config.Commands.json
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:971)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)
   [junit4]   2> 	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:745)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:726)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:507)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: Unable to create core [collection2_shard6_replica_n25]
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1018)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:938)
   [junit4]   2> 	... 34 more
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: invalid API spec: apispec/core.config.Commands.json
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:988)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:843)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1002)
   [junit4]   2> 	... 35 more
   [junit4]   2> Caused by: java.lang.RuntimeException: invalid API spec: apispec/core.config.Commands.json
   [junit4]   2> 	at org.apache.solr.common.util.ValidatingJsonMap.parse(ValidatingJsonMap.java:318)
   [junit4]   2> 	at org.apache.solr.common.util.Utils.lambda$getSpec$0(Utils.java:394)
   [junit4]   2> 	at org.apache.solr.api.Api.getSpec(Api.java:65)
   [junit4]   2> 	at org.apache.solr.api.ApiBag.register(ApiBag.java:73)
   [junit4]   2> 	at org.apache.solr.core.PluginBag.put(PluginBag.java:217)
   [junit4]   2> 	at org.apache.solr.core.PluginBag.init(PluginBag.java:274)
   [junit4]   2> 	at org.apache.solr.core.RequestHandlers.initHandlersFromConfig(RequestHandlers.java:130)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:946)
   [junit4]   2> 	... 37 more
   [junit4]   2> 
   [junit4]   2> 1577709 ERROR (OverseerThreadFactory-6790-thread-5-processing-n:127.0.0.1:63605_) [n:127.0.0.1:63605_    ] o.a.s.c.OverseerCollectionMessageHandler Error from shard: http://127.0.0.1:63657
   [junit4]   2> org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:63657: Error CREATEing SolrCore 'collection2_shard6_replica_n25': Unable to create core [collection2_shard6_replica_n25] Caused by: invalid API spec: apispec/core.config.Commands.json
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:627)
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
   [junit4]   2> 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
   [junit4]   2> 	at org.apache.solr.handler.component.HttpShardHandler.lambda$submit$0(HttpShardHandler.java:172)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1577727 ERROR (qtp876805094-15832) [n:127.0.0.1:63605_ c:collection2 s:shard4  x:collection2_shard4_replica_n15] o.a.s.h.RequestHandlerBase org.apache.solr.common.SolrException: Error CREATEing SolrCore 'collection2_shard4_replica_n15': Unable to create core [collection2_shard4_replica_n15] Caused by: invalid API spec: apispec/core.SchemaEdit.json
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:971)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)
   [junit4]   2> 	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:745)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:726)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:507)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: Unable to create core [collection2_shard4_replica_n15]
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1018)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:938)
   [junit4]   2> 	... 34 more
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: invalid API spec: apispec/core.SchemaEdit.json
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:988)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:843)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1002)
   [junit4]   2> 	... 35 more
   [junit4]   2> Caused by: java.lang.RuntimeException: invalid API spec: apispec/core.SchemaEdit.json
   [junit4]   2> 	at org.apache.solr.common.util.ValidatingJsonMap.parse(ValidatingJsonMap.java:318)
   [junit4]   2> 	at org.apache.solr.common.util.Utils.lambda$getSpec$0(Utils.java:394)
   [junit4]   2> 	at org.apache.solr.api.Api.getSpec(Api.java:65)
   [junit4]   2> 	at org.apache.solr.api.ApiBag.register(ApiBag.java:73)
   [junit4]   2> 	at org.apache.solr.core.PluginBag.put(PluginBag.java:217)
   [junit4]   2> 	at org.apache.solr.core.PluginBag.init(PluginBag.java:274)
   [junit4]   2> 	at org.apache.solr.core.RequestHandlers.initHandlersFromConfig(RequestHandlers.java:130)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:946)
   [junit4]   2> 	... 37 more
   [junit4]   2> 
   [junit4]   2> 1577728 ERROR (OverseerThreadFactory-6790-thread-5-processing-n:127.0.0.1:63605_) [n:127.0.0.1:63605_    ] o.a.s.c.OverseerCollectionMessageHandler Error from shard: http://127.0.0.1:63605
   [junit4]   2> org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:63605: Error CREATEing SolrCore 'collection2_shard4_replica_n15': Unable to create core [collection2_shard4_replica_n15] Caused by: invalid API spec: apispec/core.SchemaEdit.json
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:627)
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
   [junit4]   2> 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
   [junit4]   2> 	at org.apache.solr.handler.component.HttpShardHandler.lambda$submit$0(HttpShardHandler.java:172)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1577749 ERROR (qtp1355524555-15902) [n:127.0.0.1:63627_ c:collection2 s:shard7  x:collection2_shard7_replica_n27] o.a.s.h.RequestHandlerBase org.apache.solr.common.SolrException: Error CREATEing SolrCore 'collection2_shard7_replica_n27': Unable to create core [collection2_shard7_replica_n27] Caused by: invalid API spec: apispec/core.SchemaEdit.json
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:971)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)
   [junit4]   2> 	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:745)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:726)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:507)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: Unable to create core [collection2_shard7_replica_n27]
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1018)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:938)
   [junit4]   2> 	... 34 more
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: invalid API spec: apispec/core.SchemaEdit.json
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:988)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:843)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1002)
   [junit4]   2> 	... 35 more
   [junit4]   2> Caused by: java.lang.RuntimeException: invalid API spec: apispec/core.SchemaEdit.json
   [junit4]   2> 	at org.apache.solr.common.util.ValidatingJsonMap.parse(ValidatingJsonMap.java:318)
   [junit4]   2> 	at org.apache.solr.common.util.Utils.lambda$getSpec$0(Utils.java:394)
   [junit4]   2> 	at org.apache.solr.api.Api.getSpec(Api.java:65)
   [junit4]   2> 	at org.apache.solr.api.ApiBag.register(ApiBag.java:73)
   [junit4]   2> 	at org.apache.solr.core.PluginBag.put(PluginBag.java:217)
   [junit4]   2> 	at org.apache.solr.core.PluginBag.init(PluginBag.java:274)
   [junit4]   2> 	at org.apache.solr.core.RequestHandlers.initHandlersFromConfig(RequestHandlers.java:130)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:946)
   [junit4]   2> 	... 37 more
   [junit4]   2> 
   [junit4]   2> 1577749 ERROR (OverseerThreadFactory-6790-thread-5-processing-n:127.0.0.1:63605_) [n:127.0.0.1:63605_    ] o.a.s.c.OverseerCollectionMessageHandler Error from shard: http://127.0.0.1:63627
   [junit4]   2> org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:63627: Error CREATEing SolrCore 'collection2_shard7_replica_n27': Unable to create core [collection2_shard7_replica_n27] Caused by: invalid API spec: apispec/core.SchemaEdit.json
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:627)
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
   [junit4]   2> 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
   [junit4]   2> 	at org.apache.solr.handler.component.HttpShardHandler.lambda$submit$0(HttpShardHandler.java:172)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1577750 ERROR (qtp178589531-15872) [n:127.0.0.1:63617_ c:collection2 s:shard5  x:collection2_shard5_replica_n21] o.a.s.h.RequestHandlerBase org.apache.solr.common.SolrException: Error CREATEing SolrCore 'collection2_shard5_replica_n21': Unable to create core [collection2_shard5_replica_n21] Caused by: invalid API spec: apispec/core.SchemaEdit.json
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:971)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)
   [junit4]   2> 	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:745)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:726)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:507)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: Unable to create core [collection2_shard5_replica_n21]
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1018)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:938)
   [junit4]   2> 	... 34 more
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: invalid API spec: apispec/core.SchemaEdit.json
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:988)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:843)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1002)
   [junit4]   2> 	... 35 more
   [junit4]   2> Caused by: java.lang.RuntimeException: invalid API spec: apispec/core.SchemaEdit.json
   [junit4]   2> 	at org.apache.solr.common.util.ValidatingJsonMap.parse(ValidatingJsonMap.java:318)
   [junit4]   2> 	at org.apache.solr.common.util.Utils.lambda$getSpec$0(Utils.java:394)
   [junit4]   2> 	at org.apache.solr.api.Api.getSpec(Api.java:65)
   [junit4]   2> 	at org.apache.solr.api.ApiBag.register(ApiBag.java:73)
   [junit4]   2> 	at org.apache.solr.core.PluginBag.put(PluginBag.java:217)
   [junit4]   2> 	at org.apache.solr.core.PluginBag.init(PluginBag.java:274)
   [junit4]   2> 	at org.apache.solr.core.RequestHandlers.initHandlersFromConfig(RequestHandlers.java:130)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:946)
   [junit4]   2> 	... 37 more
   [junit4]   2> 
   [junit4]   2> 1577751 ERROR (OverseerThreadFactory-6790-thread-5-processing-n:127.0.0.1:63605_) [n:127.0.0.1:63605_    ] o.a.s.c.OverseerCollectionMessageHandler Error from shard: http://127.0.0.1:63617
   [junit4]   2> org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:63617: Error CREATEing SolrCore 'collection2_shard5_replica_n21': Unable to create core [collection2_shard5_replica_n21] Caused by: invalid API spec: apispec/core.SchemaEdit.json
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:627)
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
   [junit4]   2> 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
   [junit4]   2> 	at org.apache.solr.handler.component.HttpShardHandler.lambda$submit$0(HttpShardHandler.java:172)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1577751 ERROR (qtp1734619594-15957) [n:127.0.0.1:63657_ c:collection2 s:shard5  x:collection2_shard5_replica_n18] o.a.s.h.RequestHandlerBase org.apache.solr.common.SolrException: Error CREATEing SolrCore 'collection2_shard5_replica_n18': Unable to create core [collection2_shard5_replica_n18] Caused by: invalid API spec: apispec/core.config.Commands.json
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:971)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)
   [junit4]   2> 	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:745)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:726)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:507)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: Unable to create core [collection2_shard5_replica_n18]
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1018)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:938)
   [junit4]   2> 	... 34 more
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: invalid API spec: apispec/core.config.Commands.json
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:988)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:843)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1002)
   [junit4]   2> 	... 35 more
   [junit4]   2> Caused by: java.lang.RuntimeException: invalid API spec: apispec/core.config.Commands.json
   [junit4]   2> 	at org.apache.solr.common.util.ValidatingJsonMap.parse(ValidatingJsonMap.java:318)
   [junit4]   2> 	at org.apache.solr.common.util.Utils.lambda$getSpec$0(Utils.java:394)
   [junit4]   2> 	at org.apache.solr.api.Api.getSpec(Api.java:65)
   [junit4]   2> 	at org.apache.solr.api.ApiBag.register(ApiBag.java:73)
   [junit4]   2> 	at org.apache.solr.core.PluginBag.put(PluginBag.java:217)
   [junit4]   2> 	at org.apache.solr.core.PluginBag.init(PluginBag.java:274)
   [junit4]   2> 	at org.apache.solr.core.RequestHandlers.initHandlersFromConfig(RequestHandlers.java:130)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:946)
   [junit4]   2> 	... 37 more
   [junit4]   2> 
   [junit4]   2> 1577752 ERROR (OverseerThreadFactory-6790-thread-5-processing-n:127.0.0.1:63605_) [n:127.0.0.1:63605_    ] o.a.s.c.OverseerCollectionMessageHandler Error from shard: http://127.0.0.1:63657
   [junit4]   2> org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:63657: Error CREATEing SolrCore 'collection2_shard5_replica_n18': Unable to create core [collection2_shard5_replica_n18] Caused by: invalid API spec: apispec/core.config.Commands.json
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:627)
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
   [junit4]   2> 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
   [junit4]   2> 	at org.apache.solr.handler.component.HttpShardHandler.lambda$submit$0(HttpShardHandler.java:172)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1577752 ERROR (qtp1734619594-15964) [n:127.0.0.1:63657_ c:collection2 s:shard3  x:collection2_shard3_replica_n10] o.a.s.h.RequestHandlerBase org.apache.solr.common.SolrException: Error CREATEing SolrCore 'collection2_shard3_replica_n10': Unable to create core [collection2_shard3_replica_n10] Caused by: invalid API spec: apispec/core.SchemaEdit.json
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:971)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)
   [junit4]   2> 	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:745)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:726)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:507)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: Unable to create core [collection2_shard3_replica_n10]
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1018)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:938)
   [junit4]   2> 	... 34 more
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: invalid API spec: apispec/core.SchemaEdit.json
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:988)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:843)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1002)
   [junit4]   2> 	... 35 more
   [junit4]   2> Caused by: java.lang.RuntimeException: invalid API spec: apispec/core.SchemaEdit.json
   [junit4]   2> 	at org.apache.solr.common.util.ValidatingJsonMap.parse(ValidatingJsonMap.java:318)
   [junit4]   2> 	at org.apache.solr.common.util.Utils.lambda$getSpec$0(Utils.java:394)
   [junit4]   2> 	at org.apache.solr.api.Api.getSpec(Api.java:65)
   [junit4]   2> 	at org.apache.solr.api.ApiBag.register(ApiBag.java:73)
   [junit4]   2> 	at org.apache.solr.core.PluginBag.put(PluginBag.java:217)
   [junit4]   2> 	at org.apache.solr.core.PluginBag.init(PluginBag.java:274)
   [junit4]   2> 	at org.apache.solr.core.RequestHandlers.initHandlersFromConfig(RequestHandlers.java:130)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:946)
   [junit4]   2> 	... 37 more
   [junit4]   2> 
   [junit4]   2> 1577753 ERROR (OverseerThreadFactory-6790-thread-5-processing-n:127.0.0.1:63605_) [n:127.0.0.1:63605_    ] o.a.s.c.OverseerCollectionMessageHandler Error from shard: http://127.0.0.1:63657
   [junit4]   2> org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:63657: Error CREATEing SolrCore 'collection2_shard3_replica_n10': Unable to create core [collection2_shard3_replica_n10] Caused by: invalid API spec: apispec/core.SchemaEdit.json
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:627)
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
   [junit4]   2> 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
   [junit4]   2> 	at org.apache.solr.handler.component.HttpShardHandler.lambda$submit$0(HttpShardHandler.java:172)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1577755 ERROR (qtp1729832861-15991) [n:127.0.0.1:63679_ c:collection2 s:shard3  x:collection2_shard3_replica_n9] o.a.s.h.RequestHandlerBase org.apache.solr.common.SolrException: Error CREATEing SolrCore 'collection2_shard3_replica_n9': Unable to create core [collection2_shard3_replica_n9] Caused by: invalid API spec: apispec/emptySpec.json
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:971)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389)
   [junit4]   2> 	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)
   [junit4]   2> 	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:745)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:726)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:507)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:382)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
   [junit4]   2> 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: Unable to create core [collection2_shard3_replica_n9]
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1018)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:938)
   [junit4]   2> 	... 34 more
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: invalid API spec: apispec/emptySpec.json
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:988)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:843)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1002)
   [junit4]   2> 	... 35 more
   [junit4]   2> Caused by: java.lang.RuntimeException: invalid API spec: apispec/emptySpec.json
   [junit4]   2> 	at org.apache.solr.common.util.ValidatingJsonMap.parse(ValidatingJsonMap.java:318)
   [junit4]   2> 	at org.apache.solr.common.util.Utils.lambda$getSpec$0(Utils.java:394)
   [junit4]   2> 	at org.apache.solr.api.Api.getSpec(Api.java:65)
   [junit4]   2> 	at org.apache.solr.api.ApiBag.register(ApiBag.java:73)
   [junit4]   2> 	at org.apache.solr.core.PluginBag.put(PluginBag.java:217)
   [junit4]   2> 	at org.apache.solr.core.PluginBag.init(PluginBag.java:274)
   [junit4]   2> 	at org.apache.solr.core.RequestHandlers.initHandlersFromConfig(RequestHandlers.java:130)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:946)
   [junit4]   2> 	... 37 more
   [junit4]   2> 
   [junit4]   2> 1577756 ERROR (OverseerThreadFactory-6790-thread-5-processing-n:127.0.0.1:63605_) [n:127.0.0.1:63605_    ] o.a.s.c.OverseerCollectionMessageHandler Error from shard: http://127.0.0.1:63679
   [junit4]   2> org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:63679: Error CREATEing SolrCore 'collection2_shard3_replica_n9': Unable to create core [collection2_shard3_replica_n9] Caused by: invalid API spec: apispec/emptySpec.json
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:627)
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
   [junit4]   2> 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
   [junit4]   2> 	at org.apache.solr.handler.component.HttpShardHandler.lambda$submit$0(HttpShardHandler.java:172)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1577759 ERROR (qtp1729832861-15993) [n:127.0.0.1:63679_ c:collection2 s:shard

[...truncated too long message...]

S-ASCII
   [junit4] FAILURE 6.33s J0 | SolrSlf4jReporterTest.testReporter <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: count1=6, count2=6 - no 'solr.node' logs in: {numFound=0,start=0,docs=[]}
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([667BF9D15C701FF8:399FD4E6377C8CBD]:0)
   [junit4]    > 	at org.apache.solr.metrics.reporters.SolrSlf4jReporterTest.testReporter(SolrSlf4jReporterTest.java:90)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 2726076 INFO  (SUITE-SolrSlf4jReporterTest-seed#[667BF9D15C701FF8]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.metrics.reporters.SolrSlf4jReporterTest_667BF9D15C701FF8-001
   [junit4]   2> NOTE: test params are: codec=FastDecompressionCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=FAST_DECOMPRESSION, chunkSize=9756, maxDocsPerChunk=3, blockSize=138), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=FAST_DECOMPRESSION, chunkSize=9756, blockSize=138)), sim=RandomSimilarity(queryNorm=false): {}, locale=ar, timezone=Asia/Yangon
   [junit4]   2> NOTE: Mac OS X 10.11.6 x86_64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=3,threads=1,free=113168744,total=528482304
   [junit4]   2> NOTE: All tests run in this JVM: [MergeStrategyTest, PathHierarchyTokenizerFactoryTest, WrapperMergePolicyFactoryTest, DisMaxRequestHandlerTest, TestUninvertingReader, DirectSolrConnectionTest, TestApiFramework, TestSolrCloudWithHadoopAuthPlugin, DistanceFunctionTest, TestOnReconnectListenerSupport, DistanceUnitsTest, TestUpdate, LeaderElectionTest, TestSSLRandomization, ZkCLITest, TestCloudManagedSchema, ExitableDirectoryReaderTest, TestInfoStreamLogging, TestCursorMarkWithoutUniqueKey, FastVectorHighlighterTest, TestCSVResponseWriter, NotRequiredUniqueKeyTest, IndexBasedSpellCheckerTest, AlternateDirectoryTest, HighlighterTest, TestDistribDocBasedVersion, HdfsTlogReplayBufferedWhileIndexingTest, TestTolerantUpdateProcessorRandomCloud, TestGraphTermsQParserPlugin, TestLazyCores, DistributedDebugComponentTest, OverseerModifyCollectionTest, NumericFieldsTest, DataDrivenBlockJoinTest, TestSchemaResource, TestJsonFacetRefinement, TestConfigSetsAPIExclusivity, TestCodecSupport, TestPhraseSuggestions, EnumFieldTest, TestFoldingMultitermQuery, DirectUpdateHandlerTest, TestTlogReplica, ClusterStateUpdateTest, ZkFailoverTest, TestMinMaxOnMultiValuedField, HighlighterMaxOffsetTest, SimpleMLTQParserTest, TestStressReorder, BadCopyFieldTest, RollingRestartTest, TestWriterPerf, HttpSolrCallGetCoreTest, TestFaceting, TestIBSimilarityFactory, TestGraphMLResponseWriter, SolrMetricsIntegrationTest, BasicAuthStandaloneTest, ZkControllerTest, TestRuleBasedAuthorizationPlugin, TestFieldCacheSortRandom, TestOverriddenPrefixQueryForCustomFieldType, TestRandomCollapseQParserPlugin, TestReRankQParserPlugin, TestSolr4Spatial2, PrimitiveFieldTypeTest, TestBlobHandler, BlockJoinFacetDistribTest, BigEndianAscendingWordDeserializerTest, HdfsWriteToMultipleCollectionsTest, SolrCloudExampleTest, TestReqParamsAPI, TestSimpleQParserPlugin, ResourceLoaderTest, PeerSyncReplicationTest, TestXIncludeConfig, OverriddenZkACLAndCredentialsProvidersTest, TolerantUpdateProcessorTest, SaslZkACLProviderTest, TestBinaryResponseWriter, ReplicationFactorTest, TestDFRSimilarityFactory, TestClusterProperties, SpatialFilterTest, CdcrBootstrapTest, HdfsRecoveryZkTest, NodeMutatorTest, SolrInfoBeanTest, CleanupOldIndexTest, OverseerTest, TestSolrCoreProperties, TestLeaderElectionWithEmptyReplica, TestAnalyzeInfixSuggestions, TestDelegationWithHadoopAuth, TestFieldSortValues, CollectionReloadTest, TestJsonRequest, DistribJoinFromCollectionTest, TestRandomFlRTGCloud, TestManagedSynonymFilterFactory, DefaultValueUpdateProcessorTest, TestAuthorizationFramework, ShufflingReplicaListTransformerTest, TestFieldTypeResource, SolrIndexSplitterTest, AnalysisErrorHandlingTest, TestCollapseQParserPlugin, RecoveryZkTest, TestManagedSchema, HdfsBasicDistributedZkTest, CreateCollectionCleanupTest, ScriptEngineTest, TestFieldCacheReopen, UpdateLogTest, PeerSyncTest, DateRangeFieldTest, XmlUpdateRequestHandlerTest, TestRebalanceLeaders, TestInPlaceUpdatesDistrib, CdcrVersionReplicationTest, HLLUtilTest, TestSolrCloudSnapshots, TestFileDictionaryLookup, CollectionTooManyReplicasTest, ChaosMonkeySafeLeaderWithPullReplicasTest, ResponseLogComponentTest, MultiTermTest, InfixSuggestersTest, TestOmitPositions, CSVRequestHandlerTest, SolrCoreTest, SubstringBytesRefFilterTest, TestMaxScoreQueryParser, LeaderInitiatedRecoveryOnCommitTest, RequestLoggingTest, TemplateUpdateProcessorTest, OutputWriterTest, UtilsToolTest, TestCollationFieldDocValues, ActionThrottleTest, LeaderFailoverAfterPartitionTest, ChaosMonkeyNothingIsSafeTest, TestShardHandlerFactory, TestSchemaVersionResource, ClusterStateTest, SuggestComponentContextFilterQueryTest, CursorMarkTest, PreAnalyzedUpdateProcessorTest, TestHashPartitioner, CoreAdminHandlerTest, DateMathParserTest, UUIDFieldTest, HdfsChaosMonkeySafeLeaderTest, AnalysisAfterCoreReloadTest, TestReloadDeadlock, DirectUpdateHandlerOptimizeTest, SolrJmxReporterTest, SolrIndexMetricsTest, TestDFISimilarityFactory, MetricUtilsTest, TestSolrConfigHandlerCloud, DeleteInactiveReplicaTest, TestDocTermOrds, TestInitQParser, TestPerFieldSimilarityWithDefaultOverride, TestPayloadScoreQParserPlugin, SimpleFacetsTest, TestDynamicFieldResource, TestSolrCloudWithDelegationTokens, AtomicUpdatesTest, HdfsSyncSliceTest, ShardRoutingTest, IgnoreCommitOptimizeUpdateProcessorFactoryTest, TestSolrConfigHandler, TestDistributedStatsComponentCardinality, HighlighterConfigTest, TestFieldResource, TestSolrCLIRunExample, UnloadDistributedZkTest, SyncSliceTest, OpenCloseCoreStressTest, BasicZkTest, FullSolrCloudDistribCmdsTest, TestRandomFaceting, ZkSolrClientTest, TestDistributedSearch, DistributedSpellCheckComponentTest, TestJoin, TestReload, DistributedTermsComponentTest, TestGroupingSearch, SolrCmdDistributorTest, ConvertedLegacyTest, TestFunctionQuery, TestBadConfig, ShowFileRequestHandlerTest, DistributedQueryElevationComponentTest, SimplePostToolTest, SignatureUpdateProcessorFactoryTest, TestExtendedDismaxParser, SpellCheckCollatorTest, DocValuesTest, SuggesterTSTTest, SuggesterTest, TestStressLucene, SuggesterWFSTTest, NoCacheHeaderTest, WordBreakSolrSpellCheckerTest, TestPseudoReturnFields, TestAtomicUpdateErrorCases, TestWordDelimiterFilterFactory, SortByFunctionTest, DocValuesMultiTest, TestSolrDeletionPolicy1, XsltUpdateRequestHandlerTest, CacheHeaderTest, TestSurroundQueryParser, DirectSolrSpellCheckerTest, TestReversedWildcardFilterFactory, DocumentAnalysisRequestHandlerTest, FileBasedSpellCheckerTest, RequestHandlersTest, TestValueSourceCache, TestIndexingPerformance, RequiredFieldsTest, FieldAnalysisRequestHandlerTest, IndexSchemaRuntimeFieldTest, LoggingHandlerTest, RegexBoostProcessorTest, SolrPluginUtilsTest, IndexSchemaTest, TestJmxIntegration, TestCollationField, ReturnFieldsTest, UpdateRequestProcessorFactoryTest, MBeansHandlerTest, TestQuerySenderNoQuery, BadComponentTest, TestMergePolicyConfig, TestSolrDeletionPolicy2, TestDocSet, TestBinaryField, TestSearchPerf, MinimalSchemaTest, TestConfig, TestFuzzyAnalyzedSuggestions, TestCharFilters, EchoParamsTest, TestSweetSpotSimilarityFactory, TestPerFieldSimilarity, TestLMDirichletSimilarityFactory, TestFastWriter, TimeZoneUtilsTest, TestFastOutputStream, OpenExchangeRatesOrgProviderTest, URLClassifyProcessorTest, PluginInfoTest, TestSystemIdResolver, PrimUtilsTest, TestSuggestSpellingConverter, DateFieldTest, SpellingQueryConverterTest, RAMDirectoryFactoryTest, TestSolrJ, TestUtils, TestDocumentBuilder, SliceStateTest, SystemInfoHandlerTest, CircularListTest, DistributedMLTComponentTest, CursorPagingTest, TestHighlightDedupGrouping, TestTolerantSearch, TestEmbeddedSolrServerAdminHandler, TestEmbeddedSolrServerConstructors, TestEmbeddedSolrServerSchemaAPI, TestJettySolrRunner, AliasIntegrationTest, AssignTest, ConcurrentDeleteAndCreateCollectionTest, ConfigSetsAPITest, DeleteLastCustomShardedReplicaTest, DeleteNodeTest, DeleteReplicaTest, DeleteStatusTest, DistributedQueueTest, DistributedVersionInfoTest, HttpPartitionTest, LeaderInitiatedRecoveryOnShardRestartTest, MigrateRouteKeyTest, MoveReplicaHDFSTest, MoveReplicaTest, SolrCLIZkUtilsTest, SolrXmlInZkTest, TestCloudInspectUtil, TestCloudJSONFacetJoinDomain, TestCloudPivotFacet, TestCloudSearcherWarming, TestCollectionsAPIViaSolrCloudCluster, TestCryptoKeys, TestDeleteCollectionOnDownNodes, TestExclusionRuleCollectionAccess, TestHdfsCloudBackupRestore, TestLeaderInitiatedRecoveryThread, TestLocalFSCloudBackupRestore, TestPullReplicaErrorHandling, TestRandomRequestDistribution, TestShortCircuitedRequests, TestSolrCloudWithKerberosAlt, TestSolrCloudWithSecureImpersonation, TestStressLiveNodes, TlogReplayBufferedWhileIndexingTest, TriLevelCompositeIdRoutingTest, AutoScalingHandlerTest, TestPolicyCloud, HdfsNNFailoverTest, HdfsRecoverLeaseTest, HdfsUnloadDistributedZkTest, TestClusterStateMutator, ZkStateWriterTest, RuleEngineTest, BlobRepositoryCloudTest, ConfigureRecoveryStrategyTest, CoreSorterTest, DirectoryFactoryTest, QueryResultKeyTest, TestBackupRepositoryFactory, TestConfigOverlay, TestConfigSetProperties, TestCorePropertiesReload, TestCustomStream, TestImplicitCoreProperties, TestInitParams, TestNRTOpen, TestReloadAndDeleteDocs, TestSolrXml, TestSolrCoreSnapshots, TestConfigReload, TestSQLHandler, DistributedFacetExistsSmallTest, DistributedFacetPivotSmallAdvancedTest, DistributedFacetPivotSmallTest, DistributedQueryComponentCustomSortTest, DistributedSuggestComponentTest, TestHttpShardHandlerFactory, TestTrackingShardHandlerFactory, SolrGangliaReporterTest, SolrSlf4jReporterTest]
   [junit4] Completed [636/732 (4!)] on J0 in 6.38s, 1 test, 1 failure <<< FAILURES!

[...truncated 43589 lines...]

[JENKINS] Lucene-Solr-master-MacOSX (64bit/jdk1.8.0) - Build # 4145 - Still Unstable!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-MacOSX/4145/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseG1GC

1 tests failed.
FAILED:  org.apache.solr.metrics.reporters.SolrSlf4jReporterTest.testReporter

Error Message:
No 'solr.node' logs in: {numFound=0,start=0,docs=[]}

Stack Trace:
java.lang.AssertionError: No 'solr.node' logs in: {numFound=0,start=0,docs=[]}
	at __randomizedtesting.SeedInfo.seed([673819511C0D91C8:38DC34667701028D]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.metrics.reporters.SolrSlf4jReporterTest.testReporter(SolrSlf4jReporterTest.java:85)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 12711 lines...]
   [junit4] Suite: org.apache.solr.metrics.reporters.SolrSlf4jReporterTest
   [junit4]   2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.metrics.reporters.SolrSlf4jReporterTest_673819511C0D91C8-001/init-core-data-001
   [junit4]   2> 2954513 WARN  (SUITE-SolrSlf4jReporterTest-seed#[673819511C0D91C8]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=2 numCloses=2
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=SolrSlf4jReporterTest -Dtests.method=testReporter -Dtests.seed=673819511C0D91C8 -Dtests.slow=true -Dtests.locale=id -Dtests.timezone=Canada/Mountain -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] FAILURE 6.37s J0 | SolrSlf4jReporterTest.testReporter <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: No 'solr.node' logs in: {numFound=0,start=0,docs=[]}
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([673819511C0D91C8:38DC34667701028D]:0)
   [junit4]    > 	at org.apache.solr.metrics.reporters.SolrSlf4jReporterTest.testReporter(SolrSlf4jReporterTest.java:85)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: leaving temporary files on disk at: /Users/jenkins/workspace/Lucene-Solr-master-MacOSX/solr/build/solr-core/test/J0/temp/solr.metrics.reporters.SolrSlf4jReporterTest_673819511C0D91C8-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=584, maxMBSortInHeap=6.058596106464774, sim=RandomSimilarity(queryNorm=false): {}, locale=id, timezone=Canada/Mountain
   [junit4]   2> NOTE: Mac OS X 10.11.6 x86_64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=3,threads=1,free=233036824,total=536870912
   [junit4]   2> NOTE: All tests run in this JVM: [TestSchemaResource, ChaosMonkeySafeLeaderWithPullReplicasTest, CachingDirectoryFactoryTest, TestAuthorizationFramework, CollectionsAPISolrJTest, SyncSliceTest, BlockJoinFacetSimpleTest, DistributedFacetPivotSmallTest, TestDocTermOrds, DataDrivenBlockJoinTest, TolerantUpdateProcessorTest, TriLevelCompositeIdRoutingTest, TestLegacyNumericUtils, DeleteShardTest, TestCloudManagedSchema, TestStressRecovery, ChaosMonkeyNothingIsSafeTest, OverseerModifyCollectionTest, JSONWriterTest, TestCollationFieldDocValues, TestShardHandlerFactory, LeaderFailoverAfterPartitionTest, SoftAutoCommitTest, PeerSyncReplicationTest, LeaderElectionTest, CircularListTest, TestIndexingPerformance, TestExclusionRuleCollectionAccess, TestRandomCollapseQParserPlugin, DistributedFacetPivotLongTailTest, TestHashPartitioner, TestInitQParser, TestNRTOpen, DistanceUnitsTest, TestSolrCloudWithSecureImpersonation, DateRangeFieldTest, TestSolr4Spatial2, TestStressInPlaceUpdates, NumericFieldsTest, PropertiesRequestHandlerTest, TestLRUStatsCache, TestPolicyCloud, MetricUtilsTest, NotRequiredUniqueKeyTest, HardAutoCommitTest, TestRandomFlRTGCloud, PeerSyncTest, HighlighterMaxOffsetTest, TestLazyCores, TestXmlQParser, SaslZkACLProviderTest, SolrCoreTest, IndexBasedSpellCheckerTest, TestBlobHandler, TestMinMaxOnMultiValuedField, ExitableDirectoryReaderTest, BasicAuthStandaloneTest, HdfsWriteToMultipleCollectionsTest, TestDFISimilarityFactory, TestLMDirichletSimilarityFactory, TestSimpleQParserPlugin, HttpPartitionTest, TestIBSimilarityFactory, TestCryptoKeys, IgnoreCommitOptimizeUpdateProcessorFactoryTest, TestSolrCLIRunExample, TestSortByMinMaxFunction, BasicDistributedZkTest, IndexSchemaTest, DeleteNodeTest, TestBinaryResponseWriter, TestSolrCloudSnapshots, SolrIndexSplitterTest, TestDelegationWithHadoopAuth, TestOnReconnectListenerSupport, MultiTermTest, TestDistributedSearch, TestDistribDocBasedVersion, TestSolrCloudWithDelegationTokens, OverseerTest, DistribCursorPagingTest, TestPerFieldSimilarityWithDefaultOverride, RecoveryZkTest, TestReRankQParserPlugin, TestSolrFieldCacheBean, BlobRepositoryCloudTest, UpdateLogTest, TestHdfsBackupRestoreCore, TestNumericTerms32, RequestLoggingTest, TestShortCircuitedRequests, TestSchemaVersionResource, TestJmxIntegration, QueryResultKeyTest, HdfsDirectoryTest, TestSort, TestAnalyzeInfixSuggestions, DirectSolrConnectionTest, TimeZoneUtilsTest, QueryParsingTest, FieldAnalysisRequestHandlerTest, TestReplicationHandler, TestSolrCloudWithHadoopAuthPlugin, QueryElevationComponentTest, ActionThrottleTest, TestPseudoReturnFields, TestPushWriter, SimpleFacetsTest, TestInfoStreamLogging, CloneFieldUpdateProcessorFactoryTest, HdfsRecoveryZkTest, TestUtils, TestGraphTermsQParserPlugin, DefaultValueUpdateProcessorTest, XsltUpdateRequestHandlerTest, OverriddenZkACLAndCredentialsProvidersTest, TestRandomDVFaceting, BlockJoinFacetDistribTest, TestRuleBasedAuthorizationPlugin, SliceStateTest, TestManagedSchema, TestConfigSetsAPIExclusivity, TestExtendedDismaxParser, TestAuthenticationFramework, TestRequestForwarding, TestClusterProperties, RuleEngineTest, TestDistributedStatsComponentCardinality, TestSolrDeletionPolicy2, OverseerStatusTest, ScriptEngineTest, TestRebalanceLeaders, RegexBoostProcessorTest, TestStressUserVersions, RollingRestartTest, AtomicUpdatesTest, PreAnalyzedFieldManagedSchemaCloudTest, TestStressReorder, CleanupOldIndexTest, SolrGraphiteReporterTest, TestPHPSerializedResponseWriter, WrapperMergePolicyFactoryTest, TestMacroExpander, TestScoreJoinQPNoScore, AnalysisAfterCoreReloadTest, TestManagedSchemaThreadSafety, HdfsBasicDistributedZkTest, HdfsNNFailoverTest, TestFieldResource, ReturnFieldsTest, ShardRoutingTest, HdfsTlogReplayBufferedWhileIndexingTest, FullHLLTest, TestSolrConfigHandler, DeleteInactiveReplicaTest, TestSolrCoreProperties, AlternateDirectoryTest, CdcrBootstrapTest, TestObjectReleaseTracker, TestNumericRangeQuery64, TestFuzzyAnalyzedSuggestions, CursorMarkTest, UniqFieldsUpdateProcessorFactoryTest, ClusterStateUpdateTest, TestLeaderElectionWithEmptyReplica, CoreAdminRequestStatusTest, TestFieldTypeResource, DirectUpdateHandlerTest, TestFieldCacheWithThreads, LeaderInitiatedRecoveryOnCommitTest, TestJsonRequest, BigEndianAscendingWordDeserializerTest, TestClusterStateMutator, TestFieldSortValues, ZkControllerTest, UnloadDistributedZkTest, MinimalSchemaTest, SolrJmxReporterTest, TestSolrConfigHandlerCloud, TestFunctionQuery, TestElisionMultitermQuery, OutputWriterTest, TestLuceneMatchVersion, SpellPossibilityIteratorTest, TestCodecSupport, TestDFRSimilarityFactory, ResourceLoaderTest, URLClassifyProcessorTest, DOMUtilTest, ClusterStateTest, CursorPagingTest, SolrTestCaseJ4Test, TestCrossCoreJoin, TestCursorMarkWithoutUniqueKey, TestDistributedMissingSort, TestTolerantSearch, AliasIntegrationTest, AssignTest, CollectionStateFormat2Test, CollectionsAPIAsyncDistributedZkTest, DistribJoinFromCollectionTest, DistributedQueueTest, DistributedVersionInfoTest, LeaderFailureAfterFreshStartTest, LeaderInitiatedRecoveryOnShardRestartTest, MigrateRouteKeyTest, MissingSegmentRecoveryTest, MoveReplicaHDFSTest, MoveReplicaHDFSUlogDirTest, MoveReplicaTest, TestCloudDeleteByQuery, TestCloudInspectUtil, TestCloudPivotFacet, TestCloudSearcherWarming, TestCollectionsAPIViaSolrCloudCluster, TestDeleteCollectionOnDownNodes, TestLeaderInitiatedRecoveryThread, TestLockTree, TestPullReplica, TestRandomRequestDistribution, TestSSLRandomization, TestTlogReplica, TestImplicitCoreProperties, TestInitParams, TestReloadAndDeleteDocs, TestSolrCoreSnapshots, SearchHandlerTest, TestConfigReload, TestReqParamsAPI, TestSQLHandler, DistributedFacetPivotSmallAdvancedTest, DistributedQueryComponentCustomSortTest, DistributedSuggestComponentTest, ShufflingReplicaListTransformerTest, SuggestComponentContextFilterQueryTest, SuggestComponentTest, TestHttpShardHandlerFactory, TestPivotHelperCode, TestTrackingShardHandlerFactory, JavabinLoaderTest, TestUnifiedSolrHighlighter, TestSlowCompositeReaderWrapper, TestLegacyField, TestLegacyTerms, TestMultiValuedNumericRangeQuery, TestNumericTokenStream, JvmMetricsTest, MetricsConfigTest, SolrCoreMetricManagerTest, SolrMetricManagerTest, SolrMetricReporterTest, SolrMetricsIntegrationTest, SolrSlf4jReporterTest]
   [junit4] Completed [626/732 (1!)] on J0 in 6.42s, 1 test, 1 failure <<< FAILURES!

[...truncated 43664 lines...]