You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2017/12/05 11:20:02 UTC

[JENKINS] Lucene-Solr-7.x-MacOSX (64bit/jdk1.8.0) - Build # 329 - Still Unstable!

Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-MacOSX/329/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseParallelGC

11 tests failed.
FAILED:  org.apache.solr.cloud.CustomCollectionTest.testRouteFieldForImplicitRouter

Error Message:
Collection not found: withShardField

Stack Trace:
org.apache.solr.common.SolrException: Collection not found: withShardField
	at __randomizedtesting.SeedInfo.seed([91AB3BD76A75873D:C4FBD345C68C48CD]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:850)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:816)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
	at org.apache.solr.client.solrj.request.UpdateRequest.commit(UpdateRequest.java:233)
	at org.apache.solr.cloud.CustomCollectionTest.testRouteFieldForImplicitRouter(CustomCollectionTest.java:141)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.test

Error Message:
Could not find collection:collection2

Stack Trace:
java.lang.AssertionError: Could not find collection:collection2
	at __randomizedtesting.SeedInfo.seed([91AB3BD76A75873D:19FF040DC489EAC5]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNotNull(Assert.java:526)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:155)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:140)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:135)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:907)
	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.testIndexingBatchPerRequestWithHttpSolrClient(FullSolrCloudDistribCmdsTest.java:612)
	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.test(FullSolrCloudDistribCmdsTest.java:152)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.TestCloudDeleteByQuery.testMalformedDBQViaShard2LeaderClient

Error Message:
IOException occured when talking to server at: http://127.0.0.1:55317/solr/test_col

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:55317/solr/test_col
	at __randomizedtesting.SeedInfo.seed([91AB3BD76A75873D:BFAA7A0836D3257D]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:657)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:211)
	at org.apache.solr.cloud.TestCloudDeleteByQuery.testMalformedDBQ(TestCloudDeleteByQuery.java:203)
	at org.apache.solr.cloud.TestCloudDeleteByQuery.testMalformedDBQViaShard2LeaderClient(TestCloudDeleteByQuery.java:218)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.net.SocketException: Too many open files
	at java.net.Socket.createImpl(Socket.java:460)
	at java.net.Socket.getImpl(Socket.java:520)
	at java.net.Socket.setSoTimeout(Socket.java:1141)
	at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:120)
	at org.apache.http.impl.conn.PoolingHttpClientConnectionManager.connect(PoolingHttpClientConnectionManager.java:359)
	at org.apache.http.impl.execchain.MainClientExec.establishRoute(MainClientExec.java:381)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:237)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:542)
	... 45 more


FAILED:  org.apache.solr.cloud.TestCloudDeleteByQuery.testMalformedDBQViaNoCollectionClient

Error Message:
Error from server at http://127.0.0.1:55320/solr/test_col: Async exception during distributed update: Error from server at http://127.0.0.1:55317/solr/test_col_shard2_replica_n6: Server Error    request: http://127.0.0.1:55317/solr/test_col_shard2_replica_n6/update?update.distrib=TOLEADER&distrib.from=http%3A%2F%2F127.0.0.1%3A55320%2Fsolr%2Ftest_col_shard1_replica_n1%2F&wt=javabin&version=2 Remote error message: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudDeleteByQuery_91AB3BD76A75873D-001/tempDir-001/node2/test_col_shard2_replica_n6/data/tlog/tlog.0000000000000000002 (Too many open files)

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:55320/solr/test_col: Async exception during distributed update: Error from server at http://127.0.0.1:55317/solr/test_col_shard2_replica_n6: Server Error



request: http://127.0.0.1:55317/solr/test_col_shard2_replica_n6/update?update.distrib=TOLEADER&distrib.from=http%3A%2F%2F127.0.0.1%3A55320%2Fsolr%2Ftest_col_shard1_replica_n1%2F&wt=javabin&version=2
Remote error message: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudDeleteByQuery_91AB3BD76A75873D-001/tempDir-001/node2/test_col_shard2_replica_n6/data/tlog/tlog.0000000000000000002 (Too many open files)
	at __randomizedtesting.SeedInfo.seed([91AB3BD76A75873D:FADE8E09D4D8C921]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1103)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:883)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:816)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:895)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:858)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:873)
	at org.apache.solr.cloud.TestCloudDeleteByQuery.clearCloudCollection(TestCloudDeleteByQuery.java:196)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:968)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.TestCloudDeleteByQuery.testMalformedDBQViaShard1LeaderClient

Error Message:
Error from server at http://127.0.0.1:55320/solr/test_col: Async exception during distributed update: Error from server at http://127.0.0.1:55317/solr/test_col_shard2_replica_n6: Server Error    request: http://127.0.0.1:55317/solr/test_col_shard2_replica_n6/update?update.distrib=TOLEADER&distrib.from=http%3A%2F%2F127.0.0.1%3A55320%2Fsolr%2Ftest_col_shard1_replica_n1%2F&wt=javabin&version=2 Remote error message: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudDeleteByQuery_91AB3BD76A75873D-001/tempDir-001/node2/test_col_shard2_replica_n6/data/tlog/tlog.0000000000000000002 (Too many open files)

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:55320/solr/test_col: Async exception during distributed update: Error from server at http://127.0.0.1:55317/solr/test_col_shard2_replica_n6: Server Error



request: http://127.0.0.1:55317/solr/test_col_shard2_replica_n6/update?update.distrib=TOLEADER&distrib.from=http%3A%2F%2F127.0.0.1%3A55320%2Fsolr%2Ftest_col_shard1_replica_n1%2F&wt=javabin&version=2
Remote error message: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudDeleteByQuery_91AB3BD76A75873D-001/tempDir-001/node2/test_col_shard2_replica_n6/data/tlog/tlog.0000000000000000002 (Too many open files)
	at __randomizedtesting.SeedInfo.seed([91AB3BD76A75873D:75A9C40029B7DF64]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1103)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:883)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:816)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:895)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:858)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:873)
	at org.apache.solr.cloud.TestCloudDeleteByQuery.clearCloudCollection(TestCloudDeleteByQuery.java:196)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:968)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.TestCloudDeleteByQuery.testMalformedDBQViaShard1NonLeaderClient

Error Message:
Error from server at http://127.0.0.1:55320/solr/test_col: Async exception during distributed update: Error from server at http://127.0.0.1:55317/solr/test_col_shard2_replica_n6: Server Error    request: http://127.0.0.1:55317/solr/test_col_shard2_replica_n6/update?update.distrib=TOLEADER&distrib.from=http%3A%2F%2F127.0.0.1%3A55320%2Fsolr%2Ftest_col_shard1_replica_n1%2F&wt=javabin&version=2 Remote error message: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudDeleteByQuery_91AB3BD76A75873D-001/tempDir-001/node2/test_col_shard2_replica_n6/data/tlog/tlog.0000000000000000002 (Too many open files)

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:55320/solr/test_col: Async exception during distributed update: Error from server at http://127.0.0.1:55317/solr/test_col_shard2_replica_n6: Server Error



request: http://127.0.0.1:55317/solr/test_col_shard2_replica_n6/update?update.distrib=TOLEADER&distrib.from=http%3A%2F%2F127.0.0.1%3A55320%2Fsolr%2Ftest_col_shard1_replica_n1%2F&wt=javabin&version=2
Remote error message: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudDeleteByQuery_91AB3BD76A75873D-001/tempDir-001/node2/test_col_shard2_replica_n6/data/tlog/tlog.0000000000000000002 (Too many open files)
	at __randomizedtesting.SeedInfo.seed([91AB3BD76A75873D:4BCE79A0F46F86A]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1103)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:883)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:816)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:895)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:858)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:873)
	at org.apache.solr.cloud.TestCloudDeleteByQuery.clearCloudCollection(TestCloudDeleteByQuery.java:196)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:968)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.TestCloudDeleteByQuery.testMalformedDBQViaShard2NonLeaderClient

Error Message:
Error from server at http://127.0.0.1:55320/solr/test_col: Async exception during distributed update: Error from server at http://127.0.0.1:55317/solr/test_col_shard2_replica_n6: Server Error    request: http://127.0.0.1:55317/solr/test_col_shard2_replica_n6/update?update.distrib=TOLEADER&distrib.from=http%3A%2F%2F127.0.0.1%3A55320%2Fsolr%2Ftest_col_shard1_replica_n1%2F&wt=javabin&version=2 Remote error message: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudDeleteByQuery_91AB3BD76A75873D-001/tempDir-001/node2/test_col_shard2_replica_n6/data/tlog/tlog.0000000000000000002 (Too many open files)

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:55320/solr/test_col: Async exception during distributed update: Error from server at http://127.0.0.1:55317/solr/test_col_shard2_replica_n6: Server Error



request: http://127.0.0.1:55317/solr/test_col_shard2_replica_n6/update?update.distrib=TOLEADER&distrib.from=http%3A%2F%2F127.0.0.1%3A55320%2Fsolr%2Ftest_col_shard1_replica_n1%2F&wt=javabin&version=2
Remote error message: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudDeleteByQuery_91AB3BD76A75873D-001/tempDir-001/node2/test_col_shard2_replica_n6/data/tlog/tlog.0000000000000000002 (Too many open files)
	at __randomizedtesting.SeedInfo.seed([91AB3BD76A75873D:7CF7D8D98310CD22]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1103)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:883)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:816)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:895)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:858)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:873)
	at org.apache.solr.cloud.TestCloudDeleteByQuery.clearCloudCollection(TestCloudDeleteByQuery.java:196)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:968)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.TestCloudDeleteByQuery.testMalformedDBQViaCloudClient

Error Message:
Error from server at http://127.0.0.1:55317/solr/test_col: Async exception during distributed update: Error from server at http://127.0.0.1:55320/solr/test_col_shard1_replica_n1: Server Error    request: http://127.0.0.1:55320/solr/test_col_shard1_replica_n1/update?update.distrib=TOLEADER&distrib.from=http%3A%2F%2F127.0.0.1%3A55317%2Fsolr%2Ftest_col_shard2_replica_n6%2F&wt=javabin&version=2 Remote error message: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudDeleteByQuery_91AB3BD76A75873D-001/tempDir-001/node4/test_col_shard1_replica_n1/data/tlog/tlog.0000000000000000002 (Too many open files)

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:55317/solr/test_col: Async exception during distributed update: Error from server at http://127.0.0.1:55320/solr/test_col_shard1_replica_n1: Server Error



request: http://127.0.0.1:55320/solr/test_col_shard1_replica_n1/update?update.distrib=TOLEADER&distrib.from=http%3A%2F%2F127.0.0.1%3A55317%2Fsolr%2Ftest_col_shard2_replica_n6%2F&wt=javabin&version=2
Remote error message: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudDeleteByQuery_91AB3BD76A75873D-001/tempDir-001/node4/test_col_shard1_replica_n1/data/tlog/tlog.0000000000000000002 (Too many open files)
	at __randomizedtesting.SeedInfo.seed([91AB3BD76A75873D:CE88C6FED7C0304C]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1103)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:883)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:816)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:895)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:858)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:873)
	at org.apache.solr.cloud.TestCloudDeleteByQuery.clearCloudCollection(TestCloudDeleteByQuery.java:196)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:968)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.TestTlogReplica.testCreateDelete {seed=[91AB3BD76A75873D:3EED7667A847B421]}

Error Message:
Could not find collection : tlog_replica_test_create_delete

Stack Trace:
org.apache.solr.common.SolrException: Could not find collection : tlog_replica_test_create_delete
	at __randomizedtesting.SeedInfo.seed([91AB3BD76A75873D:3EED7667A847B421]:0)
	at org.apache.solr.common.cloud.ClusterState.getCollection(ClusterState.java:111)
	at org.apache.solr.cloud.SolrCloudTestCase.getCollectionState(SolrCloudTestCase.java:247)
	at org.apache.solr.cloud.TestTlogReplica.testCreateDelete(TestTlogReplica.java:187)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.TestTlogReplica.testOutOfOrderDBQWithInPlaceUpdates

Error Message:


Stack Trace:
java.lang.NullPointerException
	at __randomizedtesting.SeedInfo.seed([91AB3BD76A75873D:176AC33A352451DD]:0)
	at org.apache.solr.cloud.TestTlogReplica.getSolrCore(TestTlogReplica.java:862)
	at org.apache.solr.cloud.TestTlogReplica.testOutOfOrderDBQWithInPlaceUpdates(TestTlogReplica.java:661)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.TestTlogReplica.testOnlyLeaderIndexes

Error Message:


Stack Trace:
java.lang.NullPointerException
	at __randomizedtesting.SeedInfo.seed([91AB3BD76A75873D:8DAA465A1FD0F9AE]:0)
	at org.apache.solr.cloud.TestTlogReplica.getSolrCore(TestTlogReplica.java:862)
	at org.apache.solr.cloud.TestTlogReplica.testOnlyLeaderIndexes(TestTlogReplica.java:466)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 11649 lines...]
   [junit4] Suite: org.apache.solr.cloud.FullSolrCloudDistribCmdsTest
   [junit4]   2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_91AB3BD76A75873D-001/init-core-data-001
   [junit4]   2> 182894 WARN  (SUITE-FullSolrCloudDistribCmdsTest-seed#[91AB3BD76A75873D]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=8 numCloses=8
   [junit4]   2> 182895 INFO  (SUITE-FullSolrCloudDistribCmdsTest-seed#[91AB3BD76A75873D]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 182898 INFO  (SUITE-FullSolrCloudDistribCmdsTest-seed#[91AB3BD76A75873D]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776) w/ MAC_OS_X supressed clientAuth
   [junit4]   2> 182898 INFO  (SUITE-FullSolrCloudDistribCmdsTest-seed#[91AB3BD76A75873D]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 182898 INFO  (SUITE-FullSolrCloudDistribCmdsTest-seed#[91AB3BD76A75873D]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
   [junit4]   2> 182899 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[91AB3BD76A75873D]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 182899 INFO  (Thread-336) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 182899 INFO  (Thread-336) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 182902 ERROR (Thread-336) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 183000 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[91AB3BD76A75873D]) [    ] o.a.s.c.ZkTestServer start zk server on port:58910
   [junit4]   2> 183028 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[91AB3BD76A75873D]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 183032 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[91AB3BD76A75873D]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
   [junit4]   2> 183041 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[91AB3BD76A75873D]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 183046 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[91AB3BD76A75873D]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 183048 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[91AB3BD76A75873D]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 183053 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[91AB3BD76A75873D]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 183055 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[91AB3BD76A75873D]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 183057 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[91AB3BD76A75873D]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 183060 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[91AB3BD76A75873D]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 183062 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[91AB3BD76A75873D]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 183064 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[91AB3BD76A75873D]) [    ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 183067 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[91AB3BD76A75873D]) [    ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
   [junit4]   2> 183683 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[91AB3BD76A75873D]) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 183683 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[91AB3BD76A75873D]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@11992506{/,null,AVAILABLE}
   [junit4]   2> 183689 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[91AB3BD76A75873D]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@4405d60f{HTTP/1.1,[http/1.1]}{127.0.0.1:58913}
   [junit4]   2> 183690 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[91AB3BD76A75873D]) [    ] o.e.j.s.Server Started @186852ms
   [junit4]   2> 183690 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[91AB3BD76A75873D]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_91AB3BD76A75873D-001/tempDir-001/control/data, hostContext=/, hostPort=58913, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_91AB3BD76A75873D-001/control-001/cores}
   [junit4]   2> 183690 ERROR (TEST-FullSolrCloudDistribCmdsTest.test-seed#[91AB3BD76A75873D]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 183690 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[91AB3BD76A75873D]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.2.0
   [junit4]   2> 183690 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[91AB3BD76A75873D]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 183690 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[91AB3BD76A75873D]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 183690 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[91AB3BD76A75873D]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-12-05T09:59:45.216Z
   [junit4]   2> 183694 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[91AB3BD76A75873D]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 183694 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[91AB3BD76A75873D]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_91AB3BD76A75873D-001/control-001/solr.xml
   [junit4]   2> 183699 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[91AB3BD76A75873D]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 183699 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[91AB3BD76A75873D]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 183699 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[91AB3BD76A75873D]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@391e2ab0, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 183703 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[91AB3BD76A75873D]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:58910/solr
   [junit4]   2> 183858 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[91AB3BD76A75873D]) [n:127.0.0.1:58913_    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 183859 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[91AB3BD76A75873D]) [n:127.0.0.1:58913_    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:58913_
   [junit4]   2> 183861 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[91AB3BD76A75873D]) [n:127.0.0.1:58913_    ] o.a.s.c.Overseer Overseer (id=99121101827407876-127.0.0.1:58913_-n_0000000000) starting
   [junit4]   2> 183873 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[91AB3BD76A75873D]) [n:127.0.0.1:58913_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:58913_
   [junit4]   2> 183876 INFO  (zkCallback-308-thread-1-processing-n:127.0.0.1:58913_) [n:127.0.0.1:58913_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 183997 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[91AB3BD76A75873D]) [n:127.0.0.1:58913_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@391e2ab0
   [junit4]   2> 184014 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[91AB3BD76A75873D]) [n:127.0.0.1:58913_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@391e2ab0
   [junit4]   2> 184014 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[91AB3BD76A75873D]) [n:127.0.0.1:58913_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@391e2ab0
   [junit4]   2> 184015 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[91AB3BD76A75873D]) [n:127.0.0.1:58913_    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_91AB3BD76A75873D-001/control-001/cores
   [junit4]   2> 184108 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[91AB3BD76A75873D]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 184110 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[91AB3BD76A75873D]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:58910/solr ready
   [junit4]   2> 184113 INFO  (qtp222232436-1517) [n:127.0.0.1:58913_    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:58913_&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 184115 INFO  (OverseerThreadFactory-607-thread-1-processing-n:127.0.0.1:58913_) [n:127.0.0.1:58913_    ] o.a.s.c.CreateCollectionCmd Create collection control_collection
   [junit4]   2> 184228 INFO  (qtp222232436-1519) [n:127.0.0.1:58913_    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 184228 INFO  (qtp222232436-1519) [n:127.0.0.1:58913_    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 184334 INFO  (zkCallback-308-thread-1-processing-n:127.0.0.1:58913_) [n:127.0.0.1:58913_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 185249 INFO  (qtp222232436-1519) [n:127.0.0.1:58913_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
   [junit4]   2> 185258 INFO  (qtp222232436-1519) [n:127.0.0.1:58913_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
   [junit4]   2> 185315 INFO  (qtp222232436-1519) [n:127.0.0.1:58913_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 185335 INFO  (qtp222232436-1519) [n:127.0.0.1:58913_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
   [junit4]   2> 185335 INFO  (qtp222232436-1519) [n:127.0.0.1:58913_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@391e2ab0
   [junit4]   2> 185336 INFO  (qtp222232436-1519) [n:127.0.0.1:58913_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 185336 INFO  (qtp222232436-1519) [n:127.0.0.1:58913_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_91AB3BD76A75873D-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_91AB3BD76A75873D-001/control-001/cores/control_collection_shard1_replica_n1/data/]
   [junit4]   2> 185348 INFO  (qtp222232436-1519) [n:127.0.0.1:58913_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=531439747, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 185356 WARN  (qtp222232436-1519) [n:127.0.0.1:58913_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 185432 INFO  (qtp222232436-1519) [n:127.0.0.1:58913_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 185432 INFO  (qtp222232436-1519) [n:127.0.0.1:58913_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 185434 INFO  (qtp222232436-1519) [n:127.0.0.1:58913_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 185434 INFO  (qtp222232436-1519) [n:127.0.0.1:58913_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 185447 INFO  (qtp222232436-1519) [n:127.0.0.1:58913_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=23, maxMergeAtOnceExplicit=35, maxMergedSegmentMB=71.2255859375, floorSegmentMB=1.02734375, forceMergeDeletesPctAllowed=22.041071264778942, segmentsPerTier=42.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 185447 INFO  (qtp222232436-1519) [n:127.0.0.1:58913_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@7c3583ec[control_collection_shard1_replica_n1] main]
   [junit4]   2> 185452 INFO  (qtp222232436-1519) [n:127.0.0.1:58913_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 185452 INFO  (qtp222232436-1519) [n:127.0.0.1:58913_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 185453 INFO  (qtp222232436-1519) [n:127.0.0.1:58913_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 185453 INFO  (qtp222232436-1519) [n:127.0.0.1:58913_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1585937631914491904
   [junit4]   2> 185454 INFO  (searcherExecutor-610-thread-1-processing-n:127.0.0.1:58913_ x:control_collection_shard1_replica_n1 s:shard1 c:control_collection) [n:127.0.0.1:58913_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@7c3583ec[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 185459 INFO  (qtp222232436-1519) [n:127.0.0.1:58913_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 185459 INFO  (qtp222232436-1519) [n:127.0.0.1:58913_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 185459 INFO  (qtp222232436-1519) [n:127.0.0.1:58913_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:58913/control_collection_shard1_replica_n1/
   [junit4]   2> 185459 INFO  (qtp222232436-1519) [n:127.0.0.1:58913_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 185459 INFO  (qtp222232436-1519) [n:127.0.0.1:58913_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:58913/control_collection_shard1_replica_n1/ has no replicas
   [junit4]   2> 185459 INFO  (qtp222232436-1519) [n:127.0.0.1:58913_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 185463 INFO  (qtp222232436-1519) [n:127.0.0.1:58913_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:58913/control_collection_shard1_replica_n1/ shard1
   [junit4]   2> 185568 INFO  (zkCallback-308-thread-2-processing-n:127.0.0.1:58913_) [n:127.0.0.1:58913_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 185573 INFO  (qtp222232436-1519) [n:127.0.0.1:58913_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 185575 INFO  (qtp222232436-1519) [n:127.0.0.1:58913_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1347
   [junit4]   2> 185578 INFO  (qtp222232436-1517) [n:127.0.0.1:58913_    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 185679 INFO  (zkCallback-308-thread-2-processing-n:127.0.0.1:58913_) [n:127.0.0.1:58913_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 186122 INFO  (OverseerCollectionConfigSetProcessor-99121101827407876-127.0.0.1:58913_-n_0000000000) [n:127.0.0.1:58913_    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 186579 INFO  (qtp222232436-1517) [n:127.0.0.1:58913_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:58913_&wt=javabin&version=2} status=0 QTime=2466
   [junit4]   2> 186588 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[91AB3BD76A75873D]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 186589 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[91AB3BD76A75873D]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:58910/solr ready
   [junit4]   2> 186589 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[91AB3BD76A75873D]) [    ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 186590 INFO  (qtp222232436-1522) [n:127.0.0.1:58913_    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=3&createNodeSet=&stateFormat=2&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 186593 INFO  (OverseerThreadFactory-607-thread-2-processing-n:127.0.0.1:58913_) [n:127.0.0.1:58913_    ] o.a.s.c.CreateCollectionCmd Create collection collection1
   [junit4]   2> 186595 WARN  (OverseerThreadFactory-607-thread-2-processing-n:127.0.0.1:58913_) [n:127.0.0.1:58913_    ] o.a.s.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
   [junit4]   2> 186818 INFO  (qtp222232436-1522) [n:127.0.0.1:58913_    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 186820 INFO  (qtp222232436-1522) [n:127.0.0.1:58913_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=3&createNodeSet=&stateFormat=2&wt=javabin&version=2} status=0 QTime=229
   [junit4]   2> 187108 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[91AB3BD76A75873D]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_91AB3BD76A75873D-001/shard-1-001 of type NRT
   [junit4]   2> 187109 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[91AB3BD76A75873D]) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 187110 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[91AB3BD76A75873D]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2e0372be{/,null,AVAILABLE}
   [junit4]   2> 187110 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[91AB3BD76A75873D]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@2ae04930{HTTP/1.1,[http/1.1]}{127.0.0.1:58933}
   [junit4]   2> 187110 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[91AB3BD76A75873D]) [    ] o.e.j.s.Server Started @190273ms
   [junit4]   2> 187110 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[91AB3BD76A75873D]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_91AB3BD76A75873D-001/tempDir-001/jetty1, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/, hostPort=58933, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_91AB3BD76A75873D-001/shard-1-001/cores}
   [junit4]   2> 187111 ERROR (TEST-FullSolrCloudDistribCmdsTest.test-seed#[91AB3BD76A75873D]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 187111 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[91AB3BD76A75873D]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.2.0
   [junit4]   2> 187111 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[91AB3BD76A75873D]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 187111 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[91AB3BD76A75873D]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 187111 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[91AB3BD76A75873D]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-12-05T09:59:48.637Z
   [junit4]   2> 187115 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[91AB3BD76A75873D]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 187115 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[91AB3BD76A75873D]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest_91AB3BD76A75873D-001/shard-1-001/solr.xml
   [junit4]   2> 187120 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[91AB3BD76A75873D]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 187120 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[91AB3BD76A75873D]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 187120 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[91AB3BD76A75873D]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@391e2ab0, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 187123 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[91AB3BD76A75873D]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:58910/solr
   [junit4]   2> 187137 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[91AB3BD76A75873D]) [n:127.0.0.1:58933_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 187141 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[91AB3BD76A75873D]) [n:127.0.0.1:58933_    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 187143 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[91AB3BD76A75873D]) [n:127.0.0.1:58933_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:58933_
   [junit4]   2> 187145 INFO  (zkCallback-316-thread-1) [    ] o.a.s.c.c.ZkStateRea

[...truncated too long message...]

:shard2 c:test_col) [n:127.0.0.1:55317_solr c:test_col s:shard2 r:core_node8 x:test_col_shard2_replica_n6] o.a.s.c.LeaderInitiatedRecoveryThread Stop trying to send recovery command to downed replica core=test_col_shard2_replica_n4 coreNodeName=core_node7 on 127.0.0.1:55319_solr because my core container is closed.
   [junit4]   2> 2934315 INFO  (updateExecutor-5392-thread-1-processing-x:test_col_shard2_replica_n6 r:core_node8 http:////127.0.0.1:55319//solr//test_col_shard2_replica_n4// n:127.0.0.1:55317_solr s:shard2 c:test_col) [n:127.0.0.1:55317_solr c:test_col s:shard2 r:core_node8 x:test_col_shard2_replica_n6] o.a.s.c.LeaderInitiatedRecoveryThread LeaderInitiatedRecoveryThread-test_col_shard2_replica_n4 completed successfully after running for 5081.0ms
   [junit4]   2> 2934315 INFO  (jetty-closer-5384-thread-3) [    ] o.a.s.c.Overseer Overseer (id=99121281309278218-127.0.0.1:55317_solr-n_0000000000) closing
   [junit4]   2> 2934316 INFO  (OverseerStateUpdate-99121281309278218-127.0.0.1:55317_solr-n_0000000000) [n:127.0.0.1:55317_solr    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:55317_solr
   [junit4]   2> 2934316 WARN  (OverseerAutoScalingTriggerThread-99121281309278218-127.0.0.1:55317_solr-n_0000000000) [n:127.0.0.1:55317_solr    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 2934323 INFO  (jetty-closer-5384-thread-3) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@532282f8{/solr,null,UNAVAILABLE}
   [junit4]   2> 2934325 ERROR (SUITE-TestCloudDeleteByQuery-seed#[91AB3BD76A75873D]-worker) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 2934325 INFO  (SUITE-TestCloudDeleteByQuery-seed#[91AB3BD76A75873D]-worker) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:55310 55310
   [junit4]   2> 2934342 INFO  (Thread-4245) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:55310 55310
   [junit4]   2> 2934680 WARN  (Thread-4245) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	6	/solr/aliases.json
   [junit4]   2> 	6	/solr/clusterprops.json
   [junit4]   2> 	5	/solr/security.json
   [junit4]   2> 	4	/solr/configs/solrCloudCollectionConfig
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	29	/solr/collections/test_col/state.json
   [junit4]   2> 	6	/solr/clusterstate.json
   [junit4]   2> 	2	/solr/overseer_elect/election/99121281309278220-127.0.0.1:55318_solr-n_0000000001
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	6	/solr/live_nodes
   [junit4]   2> 	6	/solr/collections
   [junit4]   2> 
   [junit4]   2> NOTE: leaving temporary files on disk at: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.TestCloudDeleteByQuery_91AB3BD76A75873D-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {expected_shard_s=PostingsFormat(name=LuceneVarGapFixedInterval), id=PostingsFormat(name=Memory)}, docValues:{_version_=DocValuesFormat(name=Lucene70)}, maxPointsInLeafNode=1379, maxMBSortInHeap=6.529433756545915, sim=RandomSimilarity(queryNorm=true): {}, locale=pt-PT, timezone=Asia/Ashkhabad
   [junit4]   2> NOTE: Mac OS X 10.11.6 x86_64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=3,threads=1,free=159194136,total=526385152
   [junit4]   2> NOTE: All tests run in this JVM: [SpellCheckCollatorWithCollapseTest, SecurityConfHandlerTest, TestRawTransformer, TestEmbeddedSolrServerAdminHandler, TestTrieFacet, TestScoreJoinQPScore, TestManagedSynonymGraphFilterFactory, TestSolrCoreProperties, AutoAddReplicasIntegrationTest, UUIDFieldTest, HdfsDirectoryFactoryTest, TestCursorMarkWithoutUniqueKey, ParsingFieldUpdateProcessorsTest, PathHierarchyTokenizerFactoryTest, TestSegmentSorting, OverseerStatusTest, BJQParserTest, TestGraphMLResponseWriter, SpellCheckComponentTest, ConvertedLegacyTest, ZkCLITest, SubstringBytesRefFilterTest, TestStressRecovery, NoCacheHeaderTest, DirectUpdateHandlerTest, TestSweetSpotSimilarityFactory, TestLMDirichletSimilarityFactory, SuggesterFSTTest, TestAtomicUpdateErrorCases, SparseHLLTest, CircularListTest, TestQuerySenderListener, TestRandomDVFaceting, AsyncCallRequestStatusResponseTest, MigrateRouteKeyTest, DistributedExpandComponentTest, TestNamedUpdateProcessors, TestExpandComponent, TestReqParamsAPI, TestAddFieldRealTimeGet, TestLegacyNumericRangeQueryBuilder, TestCustomStream, TestQueryWrapperFilter, TestPointFields, TestOrdValues, RollingRestartTest, TestConfigSetsAPI, MoveReplicaTest, TestNumericTerms64, TestCodecSupport, RegexBoostProcessorTest, UnloadDistributedZkTest, TestDeleteCollectionOnDownNodes, ReturnFieldsTest, TestRecoveryHdfs, TestPivotHelperCode, TestConfigSets, GraphQueryTest, IndexBasedSpellCheckerTest, TestNoOpRegenerator, TestBulkSchemaAPI, TestTolerantUpdateProcessorRandomCloud, ChaosMonkeySafeLeaderWithPullReplicasTest, DisMaxRequestHandlerTest, BasicAuthIntegrationTest, RankQueryTest, DirectUpdateHandlerOptimizeTest, JavabinLoaderTest, PeerSyncWithIndexFingerprintCachingTest, TestHdfsUpdateLog, CoreAdminCreateDiscoverTest, StatsReloadRaceTest, HdfsThreadLeakTest, SolrXmlInZkTest, TestPushWriter, CdcrUpdateLogTest, TestFiltering, TestSurroundQueryParser, SpatialHeatmapFacetsTest, MoveReplicaHDFSFailoverTest, TlogReplayBufferedWhileIndexingTest, TestUseDocValuesAsStored2, SolrCmdDistributorTest, HLLSerializationTest, FacetPivotSmallTest, V2StandaloneTest, TestIntervalFaceting, TestSmileRequest, TestXIncludeConfig, SolrRequestParserTest, CheckHdfsIndexTest, ManagedSchemaRoundRobinCloudTest, TestConfigReload, BadIndexSchemaTest, TestLazyCores, TestEmbeddedSolrServerSchemaAPI, DistanceUnitsTest, BufferStoreTest, DateMathParserTest, TestTlogReplica, TestWriterPerf, DistributedFacetPivotSmallAdvancedTest, ReplicationFactorTest, TestImplicitCoreProperties, MinimalSchemaTest, TestReloadAndDeleteDocs, TestSystemCollAutoCreate, OverriddenZkACLAndCredentialsProvidersTest, ScriptEngineTest, DistributedFacetPivotWhiteBoxTest, SaslZkACLProviderTest, CollectionReloadTest, SolrMetricReporterTest, LeaderElectionContextKeyTest, LeaderElectionTest, TestCloudJSONFacetJoinDomain, TestSolrConfigHandlerCloud, TestPayloadCheckQParserPlugin, CSVRequestHandlerTest, PeerSyncReplicationTest, StressHdfsTest, TestFieldCollectionResource, TestConfigSetsAPIZkFailure, TestComplexPhraseLeadingWildcard, SuggesterTest, NumericFieldsTest, DeleteNodeTest, TestSQLHandler, TestDynamicFieldCollectionResource, QueryParsingTest, ZkStateReaderTest, TestInitParams, HdfsRestartWhileUpdatingTest, TestNumericRangeQuery32, TestLeaderElectionZkExpiry, TestHighlightDedupGrouping, TestSha256AuthenticationProvider, SpellCheckCollatorTest, BlockCacheTest, TestDistribIDF, TestPHPSerializedResponseWriter, ShowFileRequestHandlerTest, TestManagedStopFilterFactory, TestLegacyFieldReuse, DocExpirationUpdateProcessorFactoryTest, DistributedFacetPivotLargeTest, TestCoreDiscovery, SuggesterTSTTest, RulesTest, DistributedQueryElevationComponentTest, DeleteShardTest, BooleanFieldTest, TestSolrCoreSnapshots, TestHalfAndHalfDocValues, SignatureUpdateProcessorFactoryTest, RecoveryZkTest, BasicDistributedZkTest, CursorPagingTest, TestReplicaProperties, TestLegacyTerms, ResourceLoaderTest, TestSerializedLuceneMatchVersion, ZkControllerTest, TestStandardQParsers, TestCloudSchemaless, TestRangeQuery, VMParamsZkACLAndCredentialsProvidersTest, TestExactStatsCache, TestFreeTextSuggestions, ResponseHeaderTest, DocValuesMissingTest, UninvertDocValuesMergePolicyTest, TestBackupRepositoryFactory, SharedFSAutoReplicaFailoverTest, CollectionsAPISolrJTest, SolrCoreMetricManagerTest, DistributedTermsComponentTest, AddBlockUpdateTest, TestMissingGroups, ImplicitSnitchTest, DateFieldTest, TestJoin, BasicDistributedZk2Test, CollectionsAPIDistributedZkTest, OverseerTest, ClusterStateUpdateTest, ShardRoutingCustomTest, TestDistributedSearch, TestFaceting, TestRecovery, TestHashPartitioner, TermVectorComponentDistributedTest, TestReload, TestSolr4Spatial, TestGroupingSearch, QueryElevationComponentTest, PeerSyncTest, TestStressLucene, TestTrie, SpatialFilterTest, TestCSVLoader, SolrCoreCheckLockOnStartupTest, TestUpdate, QueryEqualityTest, DocValuesMultiTest, DistanceFunctionTest, XsltUpdateRequestHandlerTest, TestQueryUtils, DocumentAnalysisRequestHandlerTest, PrimitiveFieldTypeTest, FileBasedSpellCheckerTest, XmlUpdateRequestHandlerTest, RequestHandlersTest, DocumentBuilderTest, TermVectorComponentTest, MoreLikeThisHandlerTest, TestSolrQueryParser, SolrPluginUtilsTest, TestJmxIntegration, BinaryUpdateRequestHandlerTest, TestPartialUpdateDeduplication, TestComponentsName, SearchHandlerTest, HighlighterConfigTest, AlternateDirectoryTest, TestQuerySenderNoQuery, CopyFieldTest, ResponseLogComponentTest, SolrIndexConfigTest, BadComponentTest, TestSolrDeletionPolicy2, MultiTermTest, SampleTest, TestBinaryField, TestConfig, OutputWriterTest, TestFuzzyAnalyzedSuggestions, ExternalFileFieldSortTest, DirectSolrConnectionTest, NotRequiredUniqueKeyTest, TestPhraseSuggestions, SynonymTokenizerTest, TestPerFieldSimilarity, TestIBSimilarityFactory, TimeZoneUtilsTest, OpenExchangeRatesOrgProviderTest, PluginInfoTest, TestFastLRUCache, ChaosMonkeyNothingIsSafeTest, AssignBackwardCompatibilityTest, ChaosMonkeyNothingIsSafeWithPullReplicasTest, ConfigSetsAPITest, CreateCollectionCleanupTest, DeleteInactiveReplicaTest, DistribCursorPagingTest, DistributedVersionInfoTest, HttpPartitionTest, LeaderInitiatedRecoveryOnShardRestartTest, MoveReplicaHDFSTest, MultiThreadedOCPTest, RemoteQueryErrorTest, SimpleCollectionCreateDeleteTest, TestCloudDeleteByQuery]
   [junit4] Completed [537/751 (4!)] on J0 in 13.19s, 6 tests, 6 errors <<< FAILURES!

[...truncated 48152 lines...]